2024-11-19 12:51:29,097 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 12:51:29,108 main DEBUG Took 0.009625 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 12:51:29,109 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 12:51:29,109 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 12:51:29,110 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 12:51:29,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,118 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 12:51:29,131 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,133 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,134 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,136 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,137 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,138 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,139 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,140 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,140 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,141 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,142 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:51:29,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,144 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 12:51:29,145 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:51:29,146 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 12:51:29,148 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 12:51:29,149 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 12:51:29,150 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 12:51:29,150 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 12:51:29,159 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 12:51:29,161 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 12:51:29,163 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 12:51:29,163 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 12:51:29,164 main DEBUG createAppenders(={Console}) 2024-11-19 12:51:29,164 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 12:51:29,165 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 12:51:29,165 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 12:51:29,165 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 12:51:29,166 main DEBUG OutputStream closed 2024-11-19 12:51:29,166 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 12:51:29,166 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 12:51:29,166 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 12:51:29,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 12:51:29,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 12:51:29,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 12:51:29,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 12:51:29,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 12:51:29,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 12:51:29,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 12:51:29,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 12:51:29,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 12:51:29,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 12:51:29,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 12:51:29,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 12:51:29,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 12:51:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 12:51:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 12:51:29,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 12:51:29,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 12:51:29,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 12:51:29,253 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 12:51:29,253 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 12:51:29,254 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 12:51:29,255 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T12:51:29,485 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2 2024-11-19 12:51:29,488 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 12:51:29,488 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T12:51:29,496 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T12:51:29,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T12:51:29,545 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c, deleteOnExit=true 2024-11-19T12:51:29,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T12:51:29,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/test.cache.data in system properties and HBase conf 2024-11-19T12:51:29,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T12:51:29,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.log.dir in system properties and HBase conf 2024-11-19T12:51:29,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T12:51:29,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T12:51:29,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T12:51:29,670 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T12:51:29,778 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:51:29,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T12:51:29,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T12:51:29,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T12:51:29,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T12:51:29,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T12:51:29,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T12:51:29,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T12:51:29,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T12:51:29,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T12:51:29,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/nfs.dump.dir in system properties and HBase conf 2024-11-19T12:51:29,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/java.io.tmpdir in system properties and HBase conf 2024-11-19T12:51:29,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T12:51:29,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T12:51:29,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T12:51:30,830 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T12:51:30,910 INFO [Time-limited test {}] log.Log(170): Logging initialized @2460ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T12:51:31,001 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:51:31,067 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:51:31,093 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:51:31,093 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:51:31,095 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T12:51:31,107 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:51:31,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:51:31,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:51:31,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/java.io.tmpdir/jetty-localhost-40735-hadoop-hdfs-3_4_1-tests_jar-_-any-6910182115445873960/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T12:51:31,300 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40735} 2024-11-19T12:51:31,300 INFO [Time-limited test {}] server.Server(415): Started @2851ms 2024-11-19T12:51:31,778 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:51:31,786 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:51:31,787 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:51:31,787 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:51:31,787 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T12:51:31,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:51:31,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:51:31,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/java.io.tmpdir/jetty-localhost-35183-hadoop-hdfs-3_4_1-tests_jar-_-any-1870309357600915463/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T12:51:31,901 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35183} 2024-11-19T12:51:31,901 INFO [Time-limited test {}] server.Server(415): Started @3452ms 2024-11-19T12:51:31,950 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T12:51:32,793 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data1/current/BP-311241435-172.17.0.2-1732020690378/current, will proceed with Du for space computation calculation, 2024-11-19T12:51:32,793 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data2/current/BP-311241435-172.17.0.2-1732020690378/current, will proceed with Du for space computation calculation, 2024-11-19T12:51:32,821 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T12:51:32,864 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf1f88e1a36e071f with lease ID 0x721e25be46dc7626: Processing first storage report for DS-fe9a89e7-cc11-4986-af9e-ff777fcc3f2a from datanode DatanodeRegistration(127.0.0.1:37811, datanodeUuid=68775114-ff34-4774-8f54-36651af3c6d4, infoPort=40761, infoSecurePort=0, ipcPort=40023, storageInfo=lv=-57;cid=testClusterID;nsid=1003950939;c=1732020690378) 2024-11-19T12:51:32,866 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf1f88e1a36e071f with lease ID 0x721e25be46dc7626: from storage DS-fe9a89e7-cc11-4986-af9e-ff777fcc3f2a node DatanodeRegistration(127.0.0.1:37811, datanodeUuid=68775114-ff34-4774-8f54-36651af3c6d4, infoPort=40761, infoSecurePort=0, ipcPort=40023, storageInfo=lv=-57;cid=testClusterID;nsid=1003950939;c=1732020690378), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T12:51:32,866 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcf1f88e1a36e071f with lease ID 0x721e25be46dc7626: Processing first storage report for DS-498968b2-259c-4189-a063-8310dea99e9a from datanode DatanodeRegistration(127.0.0.1:37811, datanodeUuid=68775114-ff34-4774-8f54-36651af3c6d4, infoPort=40761, infoSecurePort=0, ipcPort=40023, storageInfo=lv=-57;cid=testClusterID;nsid=1003950939;c=1732020690378) 2024-11-19T12:51:32,866 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcf1f88e1a36e071f with lease ID 0x721e25be46dc7626: from storage DS-498968b2-259c-4189-a063-8310dea99e9a node DatanodeRegistration(127.0.0.1:37811, datanodeUuid=68775114-ff34-4774-8f54-36651af3c6d4, infoPort=40761, infoSecurePort=0, ipcPort=40023, storageInfo=lv=-57;cid=testClusterID;nsid=1003950939;c=1732020690378), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T12:51:32,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2 2024-11-19T12:51:33,024 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/zookeeper_0, clientPort=55548, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T12:51:33,033 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55548 2024-11-19T12:51:33,043 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:33,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:33,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741825_1001 (size=7) 2024-11-19T12:51:33,661 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 with version=8 2024-11-19T12:51:33,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/hbase-staging 2024-11-19T12:51:33,742 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T12:51:33,998 INFO [Time-limited test {}] client.ConnectionUtils(128): master/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:51:34,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,012 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:51:34,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:51:34,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T12:51:34,195 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T12:51:34,203 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T12:51:34,206 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:51:34,227 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 88393 (auto-detected) 2024-11-19T12:51:34,228 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T12:51:34,244 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36409 2024-11-19T12:51:34,263 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36409 connecting to ZooKeeper ensemble=127.0.0.1:55548 2024-11-19T12:51:34,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364090x0, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:51:34,367 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36409-0x101546ffad30000 connected 2024-11-19T12:51:34,452 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:34,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:34,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:51:34,469 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27, hbase.cluster.distributed=false 2024-11-19T12:51:34,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:51:34,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36409 2024-11-19T12:51:34,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36409 2024-11-19T12:51:34,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36409 2024-11-19T12:51:34,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36409 2024-11-19T12:51:34,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36409 2024-11-19T12:51:34,603 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:51:34,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,606 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:51:34,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:51:34,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:51:34,609 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:51:34,612 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:51:34,614 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35349 2024-11-19T12:51:34,617 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35349 connecting to ZooKeeper ensemble=127.0.0.1:55548 2024-11-19T12:51:34,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:34,623 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:34,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353490x0, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:51:34,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353490x0, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:51:34,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35349-0x101546ffad30001 connected 2024-11-19T12:51:34,661 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:51:34,671 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:51:34,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:51:34,679 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:51:34,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35349 2024-11-19T12:51:34,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35349 2024-11-19T12:51:34,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35349 2024-11-19T12:51:34,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35349 2024-11-19T12:51:34,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35349 2024-11-19T12:51:34,696 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aba5a916dfea:36409 2024-11-19T12:51:34,697 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aba5a916dfea,36409,1732020693843 2024-11-19T12:51:34,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:51:34,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:51:34,716 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aba5a916dfea,36409,1732020693843 2024-11-19T12:51:34,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:51:34,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:34,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:34,743 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T12:51:34,744 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aba5a916dfea,36409,1732020693843 from backup master directory 2024-11-19T12:51:34,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:51:34,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aba5a916dfea,36409,1732020693843 2024-11-19T12:51:34,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:51:34,756 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:51:34,757 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aba5a916dfea,36409,1732020693843 2024-11-19T12:51:34,759 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T12:51:34,761 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T12:51:34,819 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/hbase.id] with ID: f46e4652-6752-4a3d-82ab-95e142937d03 2024-11-19T12:51:34,820 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/.tmp/hbase.id 2024-11-19T12:51:34,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741826_1002 (size=42) 2024-11-19T12:51:35,232 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/.tmp/hbase.id]:[hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/hbase.id] 2024-11-19T12:51:35,275 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:51:35,280 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T12:51:35,300 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-19T12:51:35,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:35,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:35,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741827_1003 (size=196) 2024-11-19T12:51:35,765 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:51:35,769 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T12:51:35,785 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:35,789 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:51:35,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741828_1004 (size=1189) 2024-11-19T12:51:36,234 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store 2024-11-19T12:51:36,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741829_1005 (size=34) 2024-11-19T12:51:36,657 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T12:51:36,660 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:36,662 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:51:36,662 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:51:36,662 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:51:36,665 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:51:36,665 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:51:36,665 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:51:36,667 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020696662Disabling compacts and flushes for region at 1732020696662Disabling writes for close at 1732020696665 (+3 ms)Writing region close event to WAL at 1732020696665Closed at 1732020696665 2024-11-19T12:51:36,669 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/.initializing 2024-11-19T12:51:36,669 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/WALs/aba5a916dfea,36409,1732020693843 2024-11-19T12:51:36,677 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:51:36,691 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aba5a916dfea%2C36409%2C1732020693843, suffix=, logDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/WALs/aba5a916dfea,36409,1732020693843, archiveDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/oldWALs, maxLogs=10 2024-11-19T12:51:36,711 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/WALs/aba5a916dfea,36409,1732020693843/aba5a916dfea%2C36409%2C1732020693843.1732020696695, exclude list is [], retry=0 2024-11-19T12:51:36,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-fe9a89e7-cc11-4986-af9e-ff777fcc3f2a,DISK] 2024-11-19T12:51:36,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T12:51:36,761 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/WALs/aba5a916dfea,36409,1732020693843/aba5a916dfea%2C36409%2C1732020693843.1732020696695 2024-11-19T12:51:36,762 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40761:40761)] 2024-11-19T12:51:36,762 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:51:36,763 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:36,767 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,768 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T12:51:36,830 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:36,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:36,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T12:51:36,840 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:36,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:36,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T12:51:36,846 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:36,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:36,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T12:51:36,851 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:36,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:36,853 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,857 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,859 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,864 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,866 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,870 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:51:36,875 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:51:36,880 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:51:36,881 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68162920, jitterRate=0.015706658363342285}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:51:36,888 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732020696779Initializing all the Stores at 1732020696781 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020696781Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020696782 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020696782Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020696782Cleaning up temporary data from old regions at 1732020696866 (+84 ms)Region opened successfully at 1732020696888 (+22 ms) 2024-11-19T12:51:36,889 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T12:51:36,924 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abf47e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:51:36,955 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T12:51:36,965 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T12:51:36,965 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T12:51:36,969 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:51:36,971 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T12:51:36,976 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T12:51:36,977 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T12:51:37,003 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T12:51:37,014 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T12:51:37,074 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T12:51:37,077 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T12:51:37,079 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T12:51:37,088 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T12:51:37,090 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T12:51:37,094 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T12:51:37,104 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T12:51:37,106 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T12:51:37,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T12:51:37,131 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T12:51:37,141 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T12:51:37,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:51:37,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:51:37,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,157 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aba5a916dfea,36409,1732020693843, sessionid=0x101546ffad30000, setting cluster-up flag (Was=false) 2024-11-19T12:51:37,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,213 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T12:51:37,215 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36409,1732020693843 2024-11-19T12:51:37,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:37,257 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T12:51:37,259 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36409,1732020693843 2024-11-19T12:51:37,265 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T12:51:37,289 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(746): ClusterId : f46e4652-6752-4a3d-82ab-95e142937d03 2024-11-19T12:51:37,292 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:51:37,307 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:51:37,308 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:51:37,317 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:51:37,318 DEBUG [RS:0;aba5a916dfea:35349 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44759f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:51:37,333 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aba5a916dfea:35349 2024-11-19T12:51:37,336 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:51:37,336 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:51:37,336 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:51:37,338 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:51:37,339 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36409,1732020693843 with port=35349, startcode=1732020694570 2024-11-19T12:51:37,348 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T12:51:37,350 DEBUG [RS:0;aba5a916dfea:35349 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:51:37,355 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T12:51:37,360 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aba5a916dfea,36409,1732020693843 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T12:51:37,367 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:51:37,367 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:51:37,367 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:51:37,368 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:51:37,368 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aba5a916dfea:0, corePoolSize=10, maxPoolSize=10 2024-11-19T12:51:37,368 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,368 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:51:37,368 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,372 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732020727372 2024-11-19T12:51:37,375 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T12:51:37,376 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:51:37,377 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T12:51:37,378 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T12:51:37,382 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T12:51:37,383 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T12:51:37,383 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T12:51:37,384 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T12:51:37,384 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:37,384 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T12:51:37,384 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,394 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T12:51:37,395 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T12:51:37,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:51:37,396 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T12:51:37,398 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T12:51:37,398 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T12:51:37,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741831_1007 (size=1321) 2024-11-19T12:51:37,401 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020697400,5,FailOnTimeoutGroup] 2024-11-19T12:51:37,401 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020697401,5,FailOnTimeoutGroup] 2024-11-19T12:51:37,401 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,402 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T12:51:37,403 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,404 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,403 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36409 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:37,430 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-19T12:51:37,430 WARN [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-19T12:51:37,532 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36409,1732020693843 with port=35349, startcode=1732020694570 2024-11-19T12:51:37,534 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36409 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,537 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36409 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,545 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:51:37,546 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35543 2024-11-19T12:51:37,546 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:51:37,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:51:37,575 DEBUG [RS:0;aba5a916dfea:35349 {}] zookeeper.ZKUtil(111): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,576 WARN [RS:0;aba5a916dfea:35349 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:51:37,576 INFO [RS:0;aba5a916dfea:35349 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:51:37,576 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,578 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,35349,1732020694570] 2024-11-19T12:51:37,600 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:51:37,615 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:51:37,620 INFO [RS:0;aba5a916dfea:35349 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:51:37,620 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,621 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:51:37,626 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:51:37,627 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,627 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,627 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,627 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,628 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,629 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,629 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:51:37,629 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:51:37,629 DEBUG [RS:0;aba5a916dfea:35349 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,630 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,35349,1732020694570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:51:37,649 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:51:37,651 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,35349,1732020694570-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,651 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,651 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.Replication(171): aba5a916dfea,35349,1732020694570 started 2024-11-19T12:51:37,669 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:37,669 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,35349,1732020694570, RpcServer on aba5a916dfea/172.17.0.2:35349, sessionid=0x101546ffad30001 2024-11-19T12:51:37,670 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:51:37,670 DEBUG [RS:0;aba5a916dfea:35349 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,670 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,35349,1732020694570' 2024-11-19T12:51:37,670 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:51:37,672 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:51:37,672 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:51:37,672 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:51:37,672 DEBUG [RS:0;aba5a916dfea:35349 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,35349,1732020694570 2024-11-19T12:51:37,673 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,35349,1732020694570' 2024-11-19T12:51:37,673 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:51:37,673 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:51:37,674 DEBUG [RS:0;aba5a916dfea:35349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:51:37,674 INFO [RS:0;aba5a916dfea:35349 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:51:37,674 INFO [RS:0;aba5a916dfea:35349 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:51:37,779 INFO [RS:0;aba5a916dfea:35349 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:51:37,782 INFO [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aba5a916dfea%2C35349%2C1732020694570, suffix=, logDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570, archiveDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/oldWALs, maxLogs=32 2024-11-19T12:51:37,798 DEBUG [RS:0;aba5a916dfea:35349 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570/aba5a916dfea%2C35349%2C1732020694570.1732020697785, exclude list is [], retry=0 2024-11-19T12:51:37,802 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T12:51:37,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-fe9a89e7-cc11-4986-af9e-ff777fcc3f2a,DISK] 2024-11-19T12:51:37,803 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:51:37,806 INFO [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570/aba5a916dfea%2C35349%2C1732020694570.1732020697785 2024-11-19T12:51:37,807 DEBUG [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40761:40761)] 2024-11-19T12:51:37,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741833_1009 (size=32) 2024-11-19T12:51:38,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:38,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T12:51:38,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:51:38,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T12:51:38,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:51:38,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T12:51:38,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:51:38,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T12:51:38,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:51:38,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,235 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:51:38,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740 2024-11-19T12:51:38,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740 2024-11-19T12:51:38,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:51:38,241 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:51:38,242 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:51:38,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:51:38,247 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:51:38,248 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60206819, jitterRate=-0.10284848511219025}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:51:38,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732020698216Initializing all the Stores at 1732020698217 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698218 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698218Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020698218Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698218Cleaning up temporary data from old regions at 1732020698241 (+23 ms)Region opened successfully at 1732020698252 (+11 ms) 2024-11-19T12:51:38,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:51:38,252 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:51:38,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:51:38,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:51:38,253 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:51:38,254 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:51:38,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020698252Disabling compacts and flushes for region at 1732020698252Disabling writes for close at 1732020698252Writing region close event to WAL at 1732020698253 (+1 ms)Closed at 1732020698254 (+1 ms) 2024-11-19T12:51:38,257 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:51:38,257 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T12:51:38,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T12:51:38,271 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:51:38,274 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T12:51:38,426 DEBUG [aba5a916dfea:36409 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T12:51:38,437 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:38,443 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,35349,1732020694570, state=OPENING 2024-11-19T12:51:38,496 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T12:51:38,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:38,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:51:38,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:51:38,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:51:38,510 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:51:38,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:51:38,683 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:51:38,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37373, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:51:38,700 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T12:51:38,700 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:51:38,701 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T12:51:38,704 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aba5a916dfea%2C35349%2C1732020694570.meta, suffix=.meta, logDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570, archiveDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/oldWALs, maxLogs=32 2024-11-19T12:51:38,719 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570/aba5a916dfea%2C35349%2C1732020694570.meta.1732020698706.meta, exclude list is [], retry=0 2024-11-19T12:51:38,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-fe9a89e7-cc11-4986-af9e-ff777fcc3f2a,DISK] 2024-11-19T12:51:38,725 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/WALs/aba5a916dfea,35349,1732020694570/aba5a916dfea%2C35349%2C1732020694570.meta.1732020698706.meta 2024-11-19T12:51:38,726 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40761:40761)] 2024-11-19T12:51:38,726 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:51:38,727 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T12:51:38,730 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T12:51:38,734 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T12:51:38,737 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T12:51:38,738 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:38,738 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T12:51:38,738 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T12:51:38,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T12:51:38,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:51:38,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T12:51:38,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:51:38,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T12:51:38,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:51:38,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T12:51:38,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:51:38,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:38,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:51:38,753 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:51:38,755 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740 2024-11-19T12:51:38,760 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740 2024-11-19T12:51:38,763 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:51:38,764 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:51:38,765 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:51:38,768 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:51:38,770 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64254953, jitterRate=-0.04252658784389496}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:51:38,771 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T12:51:38,773 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732020698739Writing region info on filesystem at 1732020698739Initializing all the Stores at 1732020698741 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698741Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698741Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020698741Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020698741Cleaning up temporary data from old regions at 1732020698764 (+23 ms)Running coprocessor post-open hooks at 1732020698771 (+7 ms)Region opened successfully at 1732020698772 (+1 ms) 2024-11-19T12:51:38,780 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732020698675 2024-11-19T12:51:38,793 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T12:51:38,794 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T12:51:38,796 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:38,799 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,35349,1732020694570, state=OPEN 2024-11-19T12:51:38,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:51:38,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:51:38,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:51:38,848 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:51:38,848 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:38,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T12:51:38,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,35349,1732020694570 in 338 msec 2024-11-19T12:51:38,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T12:51:38,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 596 msec 2024-11-19T12:51:38,868 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:51:38,868 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T12:51:38,886 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:38,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:38,908 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:38,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41763, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:38,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6390 sec 2024-11-19T12:51:38,936 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732020698936, completionTime=-1 2024-11-19T12:51:38,938 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T12:51:38,938 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T12:51:38,963 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-19T12:51:38,963 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732020758963 2024-11-19T12:51:38,963 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732020818963 2024-11-19T12:51:38,963 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-19T12:51:38,965 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,965 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,965 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,967 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aba5a916dfea:36409, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,967 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,968 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:38,976 DEBUG [master/aba5a916dfea:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T12:51:38,998 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.241sec 2024-11-19T12:51:38,999 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T12:51:39,000 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T12:51:39,001 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T12:51:39,001 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T12:51:39,002 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T12:51:39,002 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:51:39,003 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T12:51:39,010 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T12:51:39,011 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T12:51:39,011 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36409,1732020693843-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:51:39,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:39,101 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T12:51:39,101 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T12:51:39,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:39,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:39,139 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f7a663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:39,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:39,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:39,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:39,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:39,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:39,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aba5a916dfea,36409,1732020693843 2024-11-19T12:51:39,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=6310 2024-11-19T12:51:39,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:51:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:51:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f9587a2 2024-11-19T12:51:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:51:39,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:51:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:51:39,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:51:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:51:39,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:51:39,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T12:51:39,244 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:39,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:51:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741835_1011 (size=1061) 2024-11-19T12:51:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:39,665 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:51:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741836_1012 (size=53) 2024-11-19T12:51:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:40,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:40,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86dc49f9832c693612c21abbffe191c4, disabling compactions & flushes 2024-11-19T12:51:40,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. after waiting 0 ms 2024-11-19T12:51:40,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86dc49f9832c693612c21abbffe191c4: Waiting for close lock at 1732020700075Disabling compacts and flushes for region at 1732020700075Disabling writes for close at 1732020700076 (+1 ms)Writing region close event to WAL at 1732020700076Closed at 1732020700076 2024-11-19T12:51:40,078 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:51:40,083 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020700079"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020700079"}]},"ts":"1732020700079"} 2024-11-19T12:51:40,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:51:40,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:51:40,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020700091"}]},"ts":"1732020700091"} 2024-11-19T12:51:40,098 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:51:40,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, ASSIGN}] 2024-11-19T12:51:40,103 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, ASSIGN 2024-11-19T12:51:40,105 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:51:40,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86dc49f9832c693612c21abbffe191c4, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:40,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, ASSIGN because future has completed 2024-11-19T12:51:40,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:51:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:40,422 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,422 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:51:40,423 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,423 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:51:40,423 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,423 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,426 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,429 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:51:40,429 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86dc49f9832c693612c21abbffe191c4 columnFamilyName A 2024-11-19T12:51:40,429 DEBUG [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:40,431 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(327): Store=86dc49f9832c693612c21abbffe191c4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:40,431 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,433 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:51:40,433 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86dc49f9832c693612c21abbffe191c4 columnFamilyName B 2024-11-19T12:51:40,433 DEBUG [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:40,434 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(327): Store=86dc49f9832c693612c21abbffe191c4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:40,434 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,436 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:51:40,437 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86dc49f9832c693612c21abbffe191c4 columnFamilyName C 2024-11-19T12:51:40,437 DEBUG [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:51:40,438 INFO [StoreOpener-86dc49f9832c693612c21abbffe191c4-1 {}] regionserver.HStore(327): Store=86dc49f9832c693612c21abbffe191c4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:51:40,438 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,438 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,440 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,441 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,443 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,443 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,444 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:51:40,448 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,452 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:51:40,454 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 86dc49f9832c693612c21abbffe191c4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70880976, jitterRate=0.05620884895324707}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:51:40,454 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:40,455 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 86dc49f9832c693612c21abbffe191c4: Running coprocessor pre-open hook at 1732020700423Writing region info on filesystem at 1732020700423Initializing all the Stores at 1732020700425 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020700425Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020700425Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020700425Cleaning up temporary data from old regions at 1732020700443 (+18 ms)Running coprocessor post-open hooks at 1732020700454 (+11 ms)Region opened successfully at 1732020700455 (+1 ms) 2024-11-19T12:51:40,458 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., pid=6, masterSystemTime=1732020700416 2024-11-19T12:51:40,461 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,461 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:40,463 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86dc49f9832c693612c21abbffe191c4, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:40,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:51:40,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T12:51:40,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 in 207 msec 2024-11-19T12:51:40,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T12:51:40,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, ASSIGN in 373 msec 2024-11-19T12:51:40,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:51:40,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020700481"}]},"ts":"1732020700481"} 2024-11-19T12:51:40,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:51:40,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:51:40,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2550 sec 2024-11-19T12:51:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:51:41,416 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:41,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,423 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d9c795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee79b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,435 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:51:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:51:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@242c798e 2024-11-19T12:51:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:51:41,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:51:41,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,449 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebcbe76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,452 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,459 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,463 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,470 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5c6971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2287fe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,480 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d3010f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adf4095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,487 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,490 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8578f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3080a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,499 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e2529f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2572aff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:51:41,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:51:41,512 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:51:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:51:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:51:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233ee7fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:51:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:51:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:51:41,514 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:51:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439eab3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:51:41,519 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,519 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,520 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,520 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,520 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,521 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,519 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,521 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,527 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:51:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:51:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:51:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,536 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:51:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,540 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:41,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:51:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T12:51:41,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:41,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:41,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:41,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:51:41,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:41,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:41,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/75f58ec12efe41b38dcc732f4cace691 is 50, key is test_row_0/A:col10/1732020701607/Put/seqid=0 2024-11-19T12:51:41,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020761706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020761713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020761718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020761713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020761739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:41,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:41,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741837_1013 (size=14341) 2024-11-19T12:51:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020761863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020761864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020761866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020761866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020761867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:41,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:41,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020762083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020762083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020762084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020762084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020762085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/75f58ec12efe41b38dcc732f4cace691 2024-11-19T12:51:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:42,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:42,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:42,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/40df6a6e9d5f491d809d698bab04a648 is 50, key is test_row_0/B:col10/1732020701607/Put/seqid=0 2024-11-19T12:51:42,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741838_1014 (size=12001) 2024-11-19T12:51:42,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/40df6a6e9d5f491d809d698bab04a648 2024-11-19T12:51:42,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/799e265efbc9471f869cf76613a97d17 is 50, key is test_row_0/C:col10/1732020701607/Put/seqid=0 2024-11-19T12:51:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741839_1015 (size=12001) 2024-11-19T12:51:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020762392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020762392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/799e265efbc9471f869cf76613a97d17 2024-11-19T12:51:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020762403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020762403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020762403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/75f58ec12efe41b38dcc732f4cace691 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691 2024-11-19T12:51:42,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691, entries=200, sequenceid=13, filesize=14.0 K 2024-11-19T12:51:42,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/40df6a6e9d5f491d809d698bab04a648 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648 2024-11-19T12:51:42,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T12:51:42,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/799e265efbc9471f869cf76613a97d17 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17 2024-11-19T12:51:42,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T12:51:42,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 86dc49f9832c693612c21abbffe191c4 in 882ms, sequenceid=13, compaction requested=false 2024-11-19T12:51:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T12:51:42,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:42,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T12:51:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:42,573 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:42,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ae83ba86f81c4c03baa59e990796a530 is 50, key is test_row_0/A:col10/1732020701705/Put/seqid=0 2024-11-19T12:51:42,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741840_1016 (size=12001) 2024-11-19T12:51:42,629 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ae83ba86f81c4c03baa59e990796a530 2024-11-19T12:51:42,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ad61dd1a2a42459eae1736cba356025a is 50, key is test_row_0/B:col10/1732020701705/Put/seqid=0 2024-11-19T12:51:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:42,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741841_1017 (size=12001) 2024-11-19T12:51:42,722 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ad61dd1a2a42459eae1736cba356025a 2024-11-19T12:51:42,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 is 50, key is test_row_0/C:col10/1732020701705/Put/seqid=0 2024-11-19T12:51:42,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741842_1018 (size=12001) 2024-11-19T12:51:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020762968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020762974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020762979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020762979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020762978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020763082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020763091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020763103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020763103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020763103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,181 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 2024-11-19T12:51:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ae83ba86f81c4c03baa59e990796a530 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530 2024-11-19T12:51:43,235 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T12:51:43,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ad61dd1a2a42459eae1736cba356025a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a 2024-11-19T12:51:43,256 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T12:51:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 2024-11-19T12:51:43,281 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T12:51:43,285 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 86dc49f9832c693612c21abbffe191c4 in 712ms, sequenceid=33, compaction requested=false 2024-11-19T12:51:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T12:51:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T12:51:43,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T12:51:43,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-19T12:51:43,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7530 sec 2024-11-19T12:51:43,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:43,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:43,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/940c99ae6aef4d069e70a424e2faaab6 is 50, key is test_row_0/A:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:43,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741843_1019 (size=12001) 2024-11-19T12:51:43,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/940c99ae6aef4d069e70a424e2faaab6 2024-11-19T12:51:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020763413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020763422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020763425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020763427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020763431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/58009f3141fc433f9484e81d03b30ac4 is 50, key is test_row_0/B:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:43,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741844_1020 (size=12001) 2024-11-19T12:51:43,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020763542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020763543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020763543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020763552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020763552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,596 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T12:51:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T12:51:43,703 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:43,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T12:51:43,708 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:43,710 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:43,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020763752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020763762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020763763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020763763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020763772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:43,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:43,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:43,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/58009f3141fc433f9484e81d03b30ac4 2024-11-19T12:51:43,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/1798cca2e4e04b5ea699a1d57e6acade is 50, key is test_row_0/C:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:44,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741845_1021 (size=12001) 2024-11-19T12:51:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:44,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/1798cca2e4e04b5ea699a1d57e6acade 2024-11-19T12:51:44,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/940c99ae6aef4d069e70a424e2faaab6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6 2024-11-19T12:51:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020764062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T12:51:44,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/58009f3141fc433f9484e81d03b30ac4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4 2024-11-19T12:51:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T12:51:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/1798cca2e4e04b5ea699a1d57e6acade as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade 2024-11-19T12:51:44,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T12:51:44,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 744ms, sequenceid=50, compaction requested=true 2024-11-19T12:51:44,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:44,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:44,117 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:44,118 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:44,121 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:44,123 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:44,124 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,125 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=35.2 K 2024-11-19T12:51:44,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:44,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:44,127 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 40df6a6e9d5f491d809d698bab04a648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732020701582 2024-11-19T12:51:44,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:44,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:44,131 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,131 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.4 K 2024-11-19T12:51:44,134 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f58ec12efe41b38dcc732f4cace691, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732020701582 2024-11-19T12:51:44,135 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae83ba86f81c4c03baa59e990796a530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732020701705 2024-11-19T12:51:44,136 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940c99ae6aef4d069e70a424e2faaab6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:44,142 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ad61dd1a2a42459eae1736cba356025a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732020701705 2024-11-19T12:51:44,143 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 58009f3141fc433f9484e81d03b30ac4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:44,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/f5c77cc538114f06b93b14576d1371b8 is 50, key is test_row_0/A:col10/1732020703426/Put/seqid=0 2024-11-19T12:51:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020764182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741846_1022 (size=12001) 2024-11-19T12:51:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020764189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020764194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,225 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#10 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:44,227 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5741fb16a2cc433fa383bf8fef9d6a72 is 50, key is test_row_0/A:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:44,229 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:44,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020764208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,235 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/742dc05c24b6406397467a6f20f5acf2 is 50, key is test_row_0/B:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741847_1023 (size=12104) 2024-11-19T12:51:44,257 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:51:44,271 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5741fb16a2cc433fa383bf8fef9d6a72 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5741fb16a2cc433fa383bf8fef9d6a72 2024-11-19T12:51:44,296 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 5741fb16a2cc433fa383bf8fef9d6a72(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:44,296 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:44,300 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020704111; duration=0sec 2024-11-19T12:51:44,301 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:44,301 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:44,301 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:44,305 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:44,305 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:44,305 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,305 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=35.2 K 2024-11-19T12:51:44,307 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 799e265efbc9471f869cf76613a97d17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732020701582 2024-11-19T12:51:44,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741848_1024 (size=12104) 2024-11-19T12:51:44,309 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4825cc0c3ede42f6ab9eb5c90fa8cc82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732020701705 2024-11-19T12:51:44,312 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1798cca2e4e04b5ea699a1d57e6acade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020764322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:44,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020764322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,341 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:44,342 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/abc754d54e474046b8806beea2bc1c14 is 50, key is test_row_0/C:col10/1732020703330/Put/seqid=0 2024-11-19T12:51:44,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020764332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020764342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741849_1025 (size=12104) 2024-11-19T12:51:44,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,392 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/abc754d54e474046b8806beea2bc1c14 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/abc754d54e474046b8806beea2bc1c14 2024-11-19T12:51:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,412 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into abc754d54e474046b8806beea2bc1c14(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:44,412 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:44,412 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=13, startTime=1732020704114; duration=0sec 2024-11-19T12:51:44,413 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:44,413 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:44,420 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T12:51:44,421 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T12:51:44,424 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:51:44,425 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T12:51:44,425 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T12:51:44,425 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T12:51:44,427 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T12:51:44,427 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T12:51:44,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020764542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020764542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020764552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020764552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020764582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/f5c77cc538114f06b93b14576d1371b8 2024-11-19T12:51:44,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/18346b74e6114493aa1dc925ad27f53d is 50, key is test_row_0/B:col10/1732020703426/Put/seqid=0 2024-11-19T12:51:44,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741850_1026 (size=12001) 2024-11-19T12:51:44,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/18346b74e6114493aa1dc925ad27f53d 2024-11-19T12:51:44,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0daa22f0c0e3419b9b7cefd3c973710f is 50, key is test_row_0/C:col10/1732020703426/Put/seqid=0 2024-11-19T12:51:44,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:44,729 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/742dc05c24b6406397467a6f20f5acf2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/742dc05c24b6406397467a6f20f5acf2 2024-11-19T12:51:44,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741851_1027 (size=12001) 2024-11-19T12:51:44,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0daa22f0c0e3419b9b7cefd3c973710f 2024-11-19T12:51:44,747 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 742dc05c24b6406397467a6f20f5acf2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:44,747 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:44,747 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020704114; duration=0sec 2024-11-19T12:51:44,748 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:44,748 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:44,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/f5c77cc538114f06b93b14576d1371b8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8 2024-11-19T12:51:44,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T12:51:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/18346b74e6114493aa1dc925ad27f53d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d 2024-11-19T12:51:44,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T12:51:44,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0daa22f0c0e3419b9b7cefd3c973710f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f 2024-11-19T12:51:44,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T12:51:44,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 684ms, sequenceid=67, compaction requested=false 2024-11-19T12:51:44,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:44,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T12:51:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:44,857 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:44,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0e2e3b9d71d24fb68c85ad16ad62e45f is 50, key is test_row_0/A:col10/1732020704204/Put/seqid=0 2024-11-19T12:51:44,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741852_1028 (size=12001) 2024-11-19T12:51:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020764904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,920 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0e2e3b9d71d24fb68c85ad16ad62e45f 2024-11-19T12:51:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020764914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020764914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020764918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ce18b15e17b0483e99079e5f8e064f8c is 50, key is test_row_0/B:col10/1732020704204/Put/seqid=0 2024-11-19T12:51:44,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741853_1029 (size=12001) 2024-11-19T12:51:44,988 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ce18b15e17b0483e99079e5f8e064f8c 2024-11-19T12:51:45,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4e82e14d25834cfcafa372b655841140 is 50, key is test_row_0/C:col10/1732020704204/Put/seqid=0 2024-11-19T12:51:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020765022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020765032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741854_1030 (size=12001) 2024-11-19T12:51:45,040 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4e82e14d25834cfcafa372b655841140 2024-11-19T12:51:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020765041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020765042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0e2e3b9d71d24fb68c85ad16ad62e45f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f 2024-11-19T12:51:45,066 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T12:51:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ce18b15e17b0483e99079e5f8e064f8c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c 2024-11-19T12:51:45,082 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T12:51:45,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/4e82e14d25834cfcafa372b655841140 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140 2024-11-19T12:51:45,099 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T12:51:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 243ms, sequenceid=85, compaction requested=true 2024-11-19T12:51:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T12:51:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T12:51:45,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T12:51:45,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-19T12:51:45,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4100 sec 2024-11-19T12:51:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:45,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/68085e82d1284f798cf94fee3e44c7bc is 50, key is test_row_0/A:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020765286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020765291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020765292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020765295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741855_1031 (size=12001) 2024-11-19T12:51:45,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/68085e82d1284f798cf94fee3e44c7bc 2024-11-19T12:51:45,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6770191535bd41b8952797ae007d314d is 50, key is test_row_0/B:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020765402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020765402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741856_1032 (size=12001) 2024-11-19T12:51:45,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6770191535bd41b8952797ae007d314d 2024-11-19T12:51:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020765414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020765414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/77d88f0ed89b452b974d339b48909d67 is 50, key is test_row_0/C:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741857_1033 (size=12001) 2024-11-19T12:51:45,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/77d88f0ed89b452b974d339b48909d67 2024-11-19T12:51:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/68085e82d1284f798cf94fee3e44c7bc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc 2024-11-19T12:51:45,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T12:51:45,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6770191535bd41b8952797ae007d314d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d 2024-11-19T12:51:45,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T12:51:45,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/77d88f0ed89b452b974d339b48909d67 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67 2024-11-19T12:51:45,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T12:51:45,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 86dc49f9832c693612c21abbffe191c4 in 319ms, sequenceid=104, compaction requested=true 2024-11-19T12:51:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:45,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:45,571 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:45,572 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:45,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:45,574 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:45,574 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:45,575 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:45,575 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/742dc05c24b6406397467a6f20f5acf2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.0 K 2024-11-19T12:51:45,576 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:45,576 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:45,576 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:45,576 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5741fb16a2cc433fa383bf8fef9d6a72, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.0 K 2024-11-19T12:51:45,577 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 742dc05c24b6406397467a6f20f5acf2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:45,580 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5741fb16a2cc433fa383bf8fef9d6a72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:45,581 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 18346b74e6114493aa1dc925ad27f53d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732020703406 2024-11-19T12:51:45,584 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c77cc538114f06b93b14576d1371b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732020703406 2024-11-19T12:51:45,585 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ce18b15e17b0483e99079e5f8e064f8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020704179 2024-11-19T12:51:45,586 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2e3b9d71d24fb68c85ad16ad62e45f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020704179 2024-11-19T12:51:45,587 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6770191535bd41b8952797ae007d314d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:45,589 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68085e82d1284f798cf94fee3e44c7bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:45,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T12:51:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:45,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/dcbb03e4aa5f459b88a83140ad35a8e8 is 50, key is test_row_0/A:col10/1732020705603/Put/seqid=0 2024-11-19T12:51:45,678 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#22 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:45,670 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#23 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:45,681 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/2083ca8bc1ac4bf084d9b56eb910f2cc is 50, key is test_row_0/A:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,682 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/96e0479bee514b3e8147b56b5bd9bf8b is 50, key is test_row_0/B:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020765674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020765674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020765674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020765680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020765683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741858_1034 (size=14341) 2024-11-19T12:51:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741859_1035 (size=12241) 2024-11-19T12:51:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741860_1036 (size=12241) 2024-11-19T12:51:45,785 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/96e0479bee514b3e8147b56b5bd9bf8b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/96e0479bee514b3e8147b56b5bd9bf8b 2024-11-19T12:51:45,803 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 96e0479bee514b3e8147b56b5bd9bf8b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:45,803 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:45,803 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=12, startTime=1732020705572; duration=0sec 2024-11-19T12:51:45,804 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:45,804 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:45,804 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:45,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020765803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,810 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:45,810 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:45,810 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:45,810 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/abc754d54e474046b8806beea2bc1c14, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.0 K 2024-11-19T12:51:45,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020765803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,811 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting abc754d54e474046b8806beea2bc1c14, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732020703302 2024-11-19T12:51:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020765804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,812 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0daa22f0c0e3419b9b7cefd3c973710f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732020703406 2024-11-19T12:51:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020765804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,813 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e82e14d25834cfcafa372b655841140, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020704179 2024-11-19T12:51:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020765805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:45,814 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d88f0ed89b452b974d339b48909d67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T12:51:45,852 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:45,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T12:51:45,857 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:45,858 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/3030ac1ba29f4f51a43de7579db490e3 is 50, key is test_row_0/C:col10/1732020704900/Put/seqid=0 2024-11-19T12:51:45,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:45,861 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:45,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:45,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741861_1037 (size=12241) 2024-11-19T12:51:45,941 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/3030ac1ba29f4f51a43de7579db490e3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/3030ac1ba29f4f51a43de7579db490e3 2024-11-19T12:51:45,956 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 3030ac1ba29f4f51a43de7579db490e3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:45,957 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:45,957 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=12, startTime=1732020705572; duration=0sec 2024-11-19T12:51:45,957 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:45,957 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:46,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020766012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020766022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020766022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020766022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020766022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/dcbb03e4aa5f459b88a83140ad35a8e8 2024-11-19T12:51:46,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/0ec08b2e3df64846b388e3fe0aa095c2 is 50, key is test_row_0/B:col10/1732020705603/Put/seqid=0 2024-11-19T12:51:46,146 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/2083ca8bc1ac4bf084d9b56eb910f2cc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/2083ca8bc1ac4bf084d9b56eb910f2cc 2024-11-19T12:51:46,158 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 2083ca8bc1ac4bf084d9b56eb910f2cc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:46,158 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:46,158 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=12, startTime=1732020705571; duration=0sec 2024-11-19T12:51:46,159 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:46,159 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:46,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741862_1038 (size=12001) 2024-11-19T12:51:46,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/0ec08b2e3df64846b388e3fe0aa095c2 2024-11-19T12:51:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:46,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/81cc21da24b443ffbe9a8c87e7f28850 is 50, key is test_row_0/C:col10/1732020705603/Put/seqid=0 2024-11-19T12:51:46,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741863_1039 (size=12001) 2024-11-19T12:51:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020766322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020766332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020766332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020766332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020766348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:46,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/81cc21da24b443ffbe9a8c87e7f28850 2024-11-19T12:51:46,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/dcbb03e4aa5f459b88a83140ad35a8e8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8 2024-11-19T12:51:46,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8, entries=200, sequenceid=118, filesize=14.0 K 2024-11-19T12:51:46,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/0ec08b2e3df64846b388e3fe0aa095c2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2 2024-11-19T12:51:46,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:46,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T12:51:46,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/81cc21da24b443ffbe9a8c87e7f28850 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850 2024-11-19T12:51:46,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T12:51:46,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 86dc49f9832c693612c21abbffe191c4 in 1073ms, sequenceid=118, compaction requested=false 2024-11-19T12:51:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:46,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T12:51:46,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:46,815 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:51:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:46,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4126943d846b445ea8255f41f6ae4df3 is 50, key is test_row_0/A:col10/1732020705675/Put/seqid=0 2024-11-19T12:51:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020766868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020766872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020766874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020766876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020766877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741864_1040 (size=12151) 2024-11-19T12:51:46,890 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4126943d846b445ea8255f41f6ae4df3 2024-11-19T12:51:46,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/c6cb15291b6f4dc48932767f5fe13998 is 50, key is test_row_0/B:col10/1732020705675/Put/seqid=0 2024-11-19T12:51:46,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741865_1041 (size=12151) 2024-11-19T12:51:46,940 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/c6cb15291b6f4dc48932767f5fe13998 2024-11-19T12:51:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/c483fdeda3ef4b6ea154370521e23c4c is 50, key is test_row_0/C:col10/1732020705675/Put/seqid=0 2024-11-19T12:51:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741866_1042 (size=12151) 2024-11-19T12:51:46,984 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/c483fdeda3ef4b6ea154370521e23c4c 2024-11-19T12:51:46,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020766992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020766992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020766992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020766993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:46,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020766993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4126943d846b445ea8255f41f6ae4df3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3 2024-11-19T12:51:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:47,013 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T12:51:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/c6cb15291b6f4dc48932767f5fe13998 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998 2024-11-19T12:51:47,023 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T12:51:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/c483fdeda3ef4b6ea154370521e23c4c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c 2024-11-19T12:51:47,036 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T12:51:47,038 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 86dc49f9832c693612c21abbffe191c4 in 224ms, sequenceid=140, compaction requested=true 2024-11-19T12:51:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:47,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:47,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T12:51:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T12:51:47,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T12:51:47,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1790 sec 2024-11-19T12:51:47,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1930 sec 2024-11-19T12:51:47,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:47,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/088055cee7cd41b6b33a03120834fb41 is 50, key is test_row_0/A:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020767251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020767253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020767255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020767256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020767259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741867_1043 (size=14541) 2024-11-19T12:51:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020767373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020767373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020767373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020767373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020767383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020767582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020767582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020767583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020767582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020767591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/088055cee7cd41b6b33a03120834fb41 2024-11-19T12:51:47,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d58a70e06ae34350a0c0f12fce147321 is 50, key is test_row_0/B:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:47,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741868_1044 (size=12151) 2024-11-19T12:51:47,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d58a70e06ae34350a0c0f12fce147321 2024-11-19T12:51:47,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/df8aeed9ce5c4c459e9daf0f6529ba15 is 50, key is test_row_0/C:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:47,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741869_1045 (size=12151) 2024-11-19T12:51:47,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/df8aeed9ce5c4c459e9daf0f6529ba15 2024-11-19T12:51:47,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/088055cee7cd41b6b33a03120834fb41 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41 2024-11-19T12:51:47,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41, entries=200, sequenceid=155, filesize=14.2 K 2024-11-19T12:51:47,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d58a70e06ae34350a0c0f12fce147321 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321 2024-11-19T12:51:47,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T12:51:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/df8aeed9ce5c4c459e9daf0f6529ba15 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15 2024-11-19T12:51:47,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T12:51:47,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 86dc49f9832c693612c21abbffe191c4 in 566ms, sequenceid=155, compaction requested=true 2024-11-19T12:51:47,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:47,778 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:47,778 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:47,780 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:47,780 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:47,780 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:47,781 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/96e0479bee514b3e8147b56b5bd9bf8b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.4 K 2024-11-19T12:51:47,781 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:47,781 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:47,781 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:47,781 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/2083ca8bc1ac4bf084d9b56eb910f2cc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=52.0 K 2024-11-19T12:51:47,782 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e0479bee514b3e8147b56b5bd9bf8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:47,782 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec08b2e3df64846b388e3fe0aa095c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732020705261 2024-11-19T12:51:47,782 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2083ca8bc1ac4bf084d9b56eb910f2cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:47,783 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcbb03e4aa5f459b88a83140ad35a8e8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732020705261 2024-11-19T12:51:47,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cb15291b6f4dc48932767f5fe13998, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732020705663 2024-11-19T12:51:47,786 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4126943d846b445ea8255f41f6ae4df3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732020705663 2024-11-19T12:51:47,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d58a70e06ae34350a0c0f12fce147321, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706871 2024-11-19T12:51:47,787 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088055cee7cd41b6b33a03120834fb41, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706865 2024-11-19T12:51:47,813 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:47,813 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a92f7625c93842a9a186a91f56e4f614 is 50, key is test_row_0/A:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:47,821 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:47,822 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a73d1fae66a14ba9841aa917b543207f is 50, key is test_row_0/B:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:47,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741870_1046 (size=12527) 2024-11-19T12:51:47,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741871_1047 (size=12527) 2024-11-19T12:51:47,899 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a73d1fae66a14ba9841aa917b543207f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a73d1fae66a14ba9841aa917b543207f 2024-11-19T12:51:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:47,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5fac15c438b3459893ad5d9fbbcd8bb6 is 50, key is test_row_0/A:col10/1732020707242/Put/seqid=0 2024-11-19T12:51:47,924 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into a73d1fae66a14ba9841aa917b543207f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:47,924 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:47,924 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=12, startTime=1732020707778; duration=0sec 2024-11-19T12:51:47,924 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:47,925 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:47,925 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:47,928 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:47,928 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:47,928 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:47,928 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/3030ac1ba29f4f51a43de7579db490e3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.4 K 2024-11-19T12:51:47,930 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3030ac1ba29f4f51a43de7579db490e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732020704900 2024-11-19T12:51:47,934 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 81cc21da24b443ffbe9a8c87e7f28850, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732020705261 2024-11-19T12:51:47,935 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c483fdeda3ef4b6ea154370521e23c4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732020705663 2024-11-19T12:51:47,935 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting df8aeed9ce5c4c459e9daf0f6529ba15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706871 2024-11-19T12:51:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020767928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020767934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020767936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020767943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020767955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:47,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741872_1048 (size=14541) 2024-11-19T12:51:47,968 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#36 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:47,970 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/a00ec95e405748dcada04d71485170f2 is 50, key is test_row_0/C:col10/1732020707209/Put/seqid=0 2024-11-19T12:51:48,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741873_1049 (size=12527) 2024-11-19T12:51:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T12:51:48,022 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:48,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T12:51:48,027 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:48,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:48,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:48,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020768052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020768061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020768062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020768062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020768071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:48,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020768271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020768272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020768271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,287 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a92f7625c93842a9a186a91f56e4f614 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a92f7625c93842a9a186a91f56e4f614 2024-11-19T12:51:48,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020768283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020768283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,307 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into a92f7625c93842a9a186a91f56e4f614(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:48,307 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:48,307 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=12, startTime=1732020707777; duration=0sec 2024-11-19T12:51:48,307 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:48,307 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:48,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5fac15c438b3459893ad5d9fbbcd8bb6 2024-11-19T12:51:48,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6c1694f2e681401e9c83729f5d6251eb is 50, key is test_row_0/B:col10/1732020707242/Put/seqid=0 2024-11-19T12:51:48,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741874_1050 (size=12151) 2024-11-19T12:51:48,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6c1694f2e681401e9c83729f5d6251eb 2024-11-19T12:51:48,422 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/a00ec95e405748dcada04d71485170f2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a00ec95e405748dcada04d71485170f2 2024-11-19T12:51:48,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bb31f9cc42bd4f6baf1ed0ade375a212 is 50, key is test_row_0/C:col10/1732020707242/Put/seqid=0 2024-11-19T12:51:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741875_1051 (size=12151) 2024-11-19T12:51:48,438 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into a00ec95e405748dcada04d71485170f2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:48,438 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:48,438 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=12, startTime=1732020707778; duration=0sec 2024-11-19T12:51:48,438 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:48,439 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:48,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bb31f9cc42bd4f6baf1ed0ade375a212 2024-11-19T12:51:48,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5fac15c438b3459893ad5d9fbbcd8bb6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6 2024-11-19T12:51:48,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6, entries=200, sequenceid=174, filesize=14.2 K 2024-11-19T12:51:48,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/6c1694f2e681401e9c83729f5d6251eb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb 2024-11-19T12:51:48,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb, entries=150, sequenceid=174, filesize=11.9 K 2024-11-19T12:51:48,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bb31f9cc42bd4f6baf1ed0ade375a212 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212 2024-11-19T12:51:48,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212, entries=150, sequenceid=174, filesize=11.9 K 2024-11-19T12:51:48,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 86dc49f9832c693612c21abbffe191c4 in 619ms, sequenceid=174, compaction requested=false 2024-11-19T12:51:48,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:48,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:48,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:48,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/8ce1c3a9147d487797d61c0ea9420de8 is 50, key is test_row_0/A:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741876_1052 (size=14541) 2024-11-19T12:51:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020768608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020768610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020768612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020768617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020768615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020768732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020768732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020768733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020768732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020768732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020768942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020768943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020768943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020768944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020768944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:48,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:48,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/8ce1c3a9147d487797d61c0ea9420de8 2024-11-19T12:51:49,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d8e90366615047b39ec1a9f9bf29af42 is 50, key is test_row_0/B:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:49,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741877_1053 (size=12151) 2024-11-19T12:51:49,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d8e90366615047b39ec1a9f9bf29af42 2024-11-19T12:51:49,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bdf1e654c28f4a84b7a85f30f0be1615 is 50, key is test_row_0/C:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:49,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741878_1054 (size=12151) 2024-11-19T12:51:49,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bdf1e654c28f4a84b7a85f30f0be1615 2024-11-19T12:51:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/8ce1c3a9147d487797d61c0ea9420de8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8 2024-11-19T12:51:49,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8, entries=200, sequenceid=192, filesize=14.2 K 2024-11-19T12:51:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d8e90366615047b39ec1a9f9bf29af42 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42 2024-11-19T12:51:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:49,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T12:51:49,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/bdf1e654c28f4a84b7a85f30f0be1615 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615 2024-11-19T12:51:49,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T12:51:49,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 599ms, sequenceid=192, compaction requested=true 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:49,183 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:49,183 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:49,186 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:49,186 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:49,186 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:49,187 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:49,187 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,187 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a92f7625c93842a9a186a91f56e4f614, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=40.6 K 2024-11-19T12:51:49,187 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,187 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a73d1fae66a14ba9841aa917b543207f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=36.0 K 2024-11-19T12:51:49,188 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92f7625c93842a9a186a91f56e4f614, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706871 2024-11-19T12:51:49,189 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a73d1fae66a14ba9841aa917b543207f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706871 2024-11-19T12:51:49,189 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c1694f2e681401e9c83729f5d6251eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732020707242 2024-11-19T12:51:49,190 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fac15c438b3459893ad5d9fbbcd8bb6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732020707242 2024-11-19T12:51:49,190 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e90366615047b39ec1a9f9bf29af42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:49,191 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ce1c3a9147d487797d61c0ea9420de8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:49,226 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:49,229 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/edf99242c3cc4a36a704880a55bc7253 is 50, key is test_row_0/B:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:49,238 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#43 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:49,239 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/45f3159613ac49f1adea7287306b3c1f is 50, key is test_row_0/A:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:49,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741879_1055 (size=12629) 2024-11-19T12:51:49,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/1ce7a36e1bd04ce99def07a5c93fc80c is 50, key is test_row_0/A:col10/1732020708613/Put/seqid=0 2024-11-19T12:51:49,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,285 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/edf99242c3cc4a36a704880a55bc7253 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/edf99242c3cc4a36a704880a55bc7253 2024-11-19T12:51:49,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020769289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020769293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741880_1056 (size=12629) 2024-11-19T12:51:49,309 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into edf99242c3cc4a36a704880a55bc7253(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:49,309 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:49,310 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020709183; duration=0sec 2024-11-19T12:51:49,310 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:49,310 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:49,310 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020769306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020769306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020769307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,325 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:49,325 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:49,325 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,326 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a00ec95e405748dcada04d71485170f2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=36.0 K 2024-11-19T12:51:49,327 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a00ec95e405748dcada04d71485170f2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732020706871 2024-11-19T12:51:49,328 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/45f3159613ac49f1adea7287306b3c1f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/45f3159613ac49f1adea7287306b3c1f 2024-11-19T12:51:49,329 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting bb31f9cc42bd4f6baf1ed0ade375a212, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732020707242 2024-11-19T12:51:49,331 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf1e654c28f4a84b7a85f30f0be1615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:49,339 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 45f3159613ac49f1adea7287306b3c1f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:49,339 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:49,339 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020709183; duration=0sec 2024-11-19T12:51:49,339 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:49,339 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:49,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741881_1057 (size=12151) 2024-11-19T12:51:49,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/1ce7a36e1bd04ce99def07a5c93fc80c 2024-11-19T12:51:49,350 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:49,351 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/511f5e1bd280415ab83c48f11018ce70 is 50, key is test_row_0/C:col10/1732020708581/Put/seqid=0 2024-11-19T12:51:49,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ed315bb508d14dcba3b176f354ce39ee is 50, key is test_row_0/B:col10/1732020708613/Put/seqid=0 2024-11-19T12:51:49,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741883_1059 (size=12151) 2024-11-19T12:51:49,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741882_1058 (size=12629) 2024-11-19T12:51:49,398 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/511f5e1bd280415ab83c48f11018ce70 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/511f5e1bd280415ab83c48f11018ce70 2024-11-19T12:51:49,408 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 511f5e1bd280415ab83c48f11018ce70(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:49,408 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:49,408 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=13, startTime=1732020709183; duration=0sec 2024-11-19T12:51:49,408 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:49,408 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020769411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020769412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020769422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020769422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020769422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020769622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020769622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020769631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020769632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020769632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:49,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ed315bb508d14dcba3b176f354ce39ee 2024-11-19T12:51:49,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/83f837abd6844406a95ebd18d66832e7 is 50, key is test_row_0/C:col10/1732020708613/Put/seqid=0 2024-11-19T12:51:49,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741884_1060 (size=12151) 2024-11-19T12:51:49,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/83f837abd6844406a95ebd18d66832e7 2024-11-19T12:51:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/1ce7a36e1bd04ce99def07a5c93fc80c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c 2024-11-19T12:51:49,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T12:51:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ed315bb508d14dcba3b176f354ce39ee as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee 2024-11-19T12:51:49,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T12:51:49,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/83f837abd6844406a95ebd18d66832e7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7 2024-11-19T12:51:49,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T12:51:49,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 86dc49f9832c693612c21abbffe191c4 in 620ms, sequenceid=211, compaction requested=false 2024-11-19T12:51:49,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:49,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T12:51:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:49,900 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:49,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0170b78980694126aa3f1536a9f25a19 is 50, key is test_row_0/A:col10/1732020709302/Put/seqid=0 2024-11-19T12:51:49,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741885_1061 (size=12151) 2024-11-19T12:51:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0170b78980694126aa3f1536a9f25a19 2024-11-19T12:51:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:49,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/267a4956449d42b4b67f6946c10c057b is 50, key is test_row_0/B:col10/1732020709302/Put/seqid=0 2024-11-19T12:51:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741886_1062 (size=12151) 2024-11-19T12:51:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020769967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020769967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:49,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020769972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:49,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020769992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020769993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020770112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020770112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020770312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020770312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020770312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020770322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020770322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,365 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/267a4956449d42b4b67f6946c10c057b 2024-11-19T12:51:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/85808c53cb7246608220e19b97d970e6 is 50, key is test_row_0/C:col10/1732020709302/Put/seqid=0 2024-11-19T12:51:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741887_1063 (size=12151) 2024-11-19T12:51:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020770622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020770622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020770622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020770631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020770631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:50,788 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/85808c53cb7246608220e19b97d970e6 2024-11-19T12:51:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0170b78980694126aa3f1536a9f25a19 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19 2024-11-19T12:51:50,811 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T12:51:50,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/267a4956449d42b4b67f6946c10c057b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b 2024-11-19T12:51:50,820 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T12:51:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/85808c53cb7246608220e19b97d970e6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6 2024-11-19T12:51:50,829 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T12:51:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 86dc49f9832c693612c21abbffe191c4 in 931ms, sequenceid=227, compaction requested=true 2024-11-19T12:51:50,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:50,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:50,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T12:51:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T12:51:50,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T12:51:50,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8040 sec 2024-11-19T12:51:50,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8150 sec 2024-11-19T12:51:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:51,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-19T12:51:51,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:51,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020709951, max timestamp=1732020711145Num uniques 859; 2024-11-19T12:51:51,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:51:51,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732020709951, max timestamp=1732020711145Num uniques 859; 2024-11-19T12:51:51,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T12:51:51,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:51,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:51,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b6b4baf41eac4995a737b785cc1174ca is 50, key is test_row_0/A:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741888_1064 (size=12151) 2024-11-19T12:51:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020771206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020771208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020771210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020771219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020771220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020771332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020771332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020771332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020771343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020771344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020771551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020771551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020771552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020771551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020771552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b6b4baf41eac4995a737b785cc1174ca 2024-11-19T12:51:51,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/24e597d0b3f64faea3e31c3050e367bd is 50, key is test_row_0/B:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741889_1065 (size=12151) 2024-11-19T12:51:51,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/24e597d0b3f64faea3e31c3050e367bd 2024-11-19T12:51:51,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/a41b04ea5e444fb4b42458bb796dc191 is 50, key is test_row_0/C:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741890_1066 (size=12151) 2024-11-19T12:51:51,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/a41b04ea5e444fb4b42458bb796dc191 2024-11-19T12:51:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b6b4baf41eac4995a737b785cc1174ca as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca 2024-11-19T12:51:51,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T12:51:51,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/24e597d0b3f64faea3e31c3050e367bd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd 2024-11-19T12:51:51,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T12:51:51,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/a41b04ea5e444fb4b42458bb796dc191 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191 2024-11-19T12:51:51,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T12:51:51,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~203.05 KB/207928, currentSize=114.05 KB/116790 for 86dc49f9832c693612c21abbffe191c4 in 545ms, sequenceid=250, compaction requested=true 2024-11-19T12:51:51,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:51,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:51,698 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:51,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:51,699 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:51,700 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:51,701 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:51,701 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:51,701 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/45f3159613ac49f1adea7287306b3c1f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.9 K 2024-11-19T12:51:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:51,702 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f3159613ac49f1adea7287306b3c1f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:51,702 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce7a36e1bd04ce99def07a5c93fc80c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732020708613 2024-11-19T12:51:51,703 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:51,703 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:51,703 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:51,703 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0170b78980694126aa3f1536a9f25a19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732020709294 2024-11-19T12:51:51,703 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/edf99242c3cc4a36a704880a55bc7253, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.9 K 2024-11-19T12:51:51,703 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b4baf41eac4995a737b785cc1174ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:51,704 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting edf99242c3cc4a36a704880a55bc7253, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:51,704 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ed315bb508d14dcba3b176f354ce39ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732020708613 2024-11-19T12:51:51,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:51,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:51,705 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 267a4956449d42b4b67f6946c10c057b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732020709294 2024-11-19T12:51:51,705 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e597d0b3f64faea3e31c3050e367bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:51,726 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:51,729 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:51,730 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d0e2c1e80fd841b5b005049a4602ee4d is 50, key is test_row_0/B:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,730 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/91c80e259ce340ef928ec09c66b3e22c is 50, key is test_row_0/A:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741891_1067 (size=12765) 2024-11-19T12:51:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741892_1068 (size=12765) 2024-11-19T12:51:51,762 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/d0e2c1e80fd841b5b005049a4602ee4d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d0e2c1e80fd841b5b005049a4602ee4d 2024-11-19T12:51:51,763 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/91c80e259ce340ef928ec09c66b3e22c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/91c80e259ce340ef928ec09c66b3e22c 2024-11-19T12:51:51,774 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into d0e2c1e80fd841b5b005049a4602ee4d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:51,774 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:51,774 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=12, startTime=1732020711698; duration=0sec 2024-11-19T12:51:51,774 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:51,775 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:51,776 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:51:51,776 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 91c80e259ce340ef928ec09c66b3e22c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:51,778 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:51,778 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=12, startTime=1732020711698; duration=0sec 2024-11-19T12:51:51,778 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:51,778 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:51,782 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:51:51,782 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:51,782 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:51,782 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/511f5e1bd280415ab83c48f11018ce70, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=47.9 K 2024-11-19T12:51:51,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 511f5e1bd280415ab83c48f11018ce70, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732020707925 2024-11-19T12:51:51,786 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f837abd6844406a95ebd18d66832e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732020708613 2024-11-19T12:51:51,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 85808c53cb7246608220e19b97d970e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732020709294 2024-11-19T12:51:51,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a41b04ea5e444fb4b42458bb796dc191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:51,815 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#56 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:51,816 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0e5ebc4c494c445f85454c2a65977a43 is 50, key is test_row_0/C:col10/1732020711146/Put/seqid=0 2024-11-19T12:51:51,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741893_1069 (size=12765) 2024-11-19T12:51:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:51,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-19T12:51:51,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020711204, max timestamp=1732020711868Num uniques 859; 2024-11-19T12:51:51,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:51,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020711204, max timestamp=1732020711867Num uniques 859; 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:51,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:51,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/6e92f4ee6b594390a3bcd2994fc720c7 is 50, key is test_row_0/A:col10/1732020711870/Put/seqid=0 2024-11-19T12:51:51,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020771919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741894_1070 (size=14741) 2024-11-19T12:51:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/6e92f4ee6b594390a3bcd2994fc720c7 2024-11-19T12:51:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020771922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020771922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020771925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020771932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:51,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/4c4fd9f1dd414011ae9be45b26333f90 is 50, key is test_row_0/B:col10/1732020711870/Put/seqid=0 2024-11-19T12:51:51,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741895_1071 (size=12301) 2024-11-19T12:51:51,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/4c4fd9f1dd414011ae9be45b26333f90 2024-11-19T12:51:52,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2ae2f5cc8e684969ac4d6bedc93d5e0d is 50, key is test_row_0/C:col10/1732020711870/Put/seqid=0 2024-11-19T12:51:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741896_1072 (size=12301) 2024-11-19T12:51:52,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2ae2f5cc8e684969ac4d6bedc93d5e0d 2024-11-19T12:51:52,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/6e92f4ee6b594390a3bcd2994fc720c7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7 2024-11-19T12:51:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020772042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020772042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020772043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020772043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7, entries=200, sequenceid=275, filesize=14.4 K 2024-11-19T12:51:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020772052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/4c4fd9f1dd414011ae9be45b26333f90 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90 2024-11-19T12:51:52,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T12:51:52,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2ae2f5cc8e684969ac4d6bedc93d5e0d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d 2024-11-19T12:51:52,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T12:51:52,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=114.05 KB/116790 for 86dc49f9832c693612c21abbffe191c4 in 204ms, sequenceid=275, compaction requested=false 2024-11-19T12:51:52,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T12:51:52,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:52,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T12:51:52,198 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T12:51:52,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:52,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:52,239 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0e5ebc4c494c445f85454c2a65977a43 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0e5ebc4c494c445f85454c2a65977a43 2024-11-19T12:51:52,248 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 0e5ebc4c494c445f85454c2a65977a43(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:52,248 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:52,249 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=12, startTime=1732020711702; duration=0sec 2024-11-19T12:51:52,249 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:52,249 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:52,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020711900, max timestamp=1732020712263Num uniques 859; 2024-11-19T12:51:52,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:51:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:52,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:51:52,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0f5cfa31292546e59b5858f208869a96 is 50, key is test_row_0/A:col10/1732020711921/Put/seqid=0 2024-11-19T12:51:52,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741897_1073 (size=12301) 2024-11-19T12:51:52,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0f5cfa31292546e59b5858f208869a96 2024-11-19T12:51:52,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/8d99e7cb77fb40fe82413e6016285efb is 50, key is test_row_0/B:col10/1732020711921/Put/seqid=0 2024-11-19T12:51:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020772292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020772296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020772297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T12:51:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020772298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020772299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741898_1074 (size=12301) 2024-11-19T12:51:52,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T12:51:52,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:52,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020772402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020772402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020772411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020772412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020772412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T12:51:52,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T12:51:52,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:52,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020772612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020772612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020772621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020772622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020772622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T12:51:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:52,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/8d99e7cb77fb40fe82413e6016285efb 2024-11-19T12:51:52,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0d0da1c180bf4b88982d1833d34f7c27 is 50, key is test_row_0/C:col10/1732020711921/Put/seqid=0 2024-11-19T12:51:52,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741899_1075 (size=12301) 2024-11-19T12:51:52,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0d0da1c180bf4b88982d1833d34f7c27 2024-11-19T12:51:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0f5cfa31292546e59b5858f208869a96 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96 2024-11-19T12:51:52,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T12:51:52,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/8d99e7cb77fb40fe82413e6016285efb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb 2024-11-19T12:51:52,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T12:51:52,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0d0da1c180bf4b88982d1833d34f7c27 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27 2024-11-19T12:51:52,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T12:51:52,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 531ms, sequenceid=298, compaction requested=true 2024-11-19T12:51:52,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:52,799 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:52,799 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:52,800 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:52,800 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:52,800 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,801 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d0e2c1e80fd841b5b005049a4602ee4d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=36.5 K 2024-11-19T12:51:52,801 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:52,801 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:52,801 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,801 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/91c80e259ce340ef928ec09c66b3e22c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=38.9 K 2024-11-19T12:51:52,802 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e2c1e80fd841b5b005049a4602ee4d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:52,802 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c80e259ce340ef928ec09c66b3e22c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:52,802 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4fd9f1dd414011ae9be45b26333f90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020711204 2024-11-19T12:51:52,803 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e92f4ee6b594390a3bcd2994fc720c7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020711204 2024-11-19T12:51:52,803 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d99e7cb77fb40fe82413e6016285efb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732020711921 2024-11-19T12:51:52,803 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5cfa31292546e59b5858f208869a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732020711921 2024-11-19T12:51:52,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T12:51:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:52,819 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:52,819 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:52,819 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/159cbedf951a4ee5939453232418adf5 is 50, key is test_row_0/B:col10/1732020711921/Put/seqid=0 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:52,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:52,819 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:52,820 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ac37dfdf5cf141d28ec15b436822c045 is 50, key is test_row_0/A:col10/1732020711921/Put/seqid=0 2024-11-19T12:51:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T12:51:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741900_1076 (size=13017) 2024-11-19T12:51:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741901_1077 (size=13017) 2024-11-19T12:51:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a76031ccf16b4a908457ea9fb0638aa7 is 50, key is test_row_0/A:col10/1732020712287/Put/seqid=0 2024-11-19T12:51:52,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741902_1078 (size=12301) 2024-11-19T12:51:52,845 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a76031ccf16b4a908457ea9fb0638aa7 2024-11-19T12:51:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/3de4d32647d64b559f49f58e1e0ff06f is 50, key is test_row_0/B:col10/1732020712287/Put/seqid=0 2024-11-19T12:51:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741903_1079 (size=12301) 2024-11-19T12:51:52,915 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/3de4d32647d64b559f49f58e1e0ff06f 2024-11-19T12:51:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/54ecbbcc43804a05a69de5e9f8bc1aa5 is 50, key is test_row_0/C:col10/1732020712287/Put/seqid=0 2024-11-19T12:51:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741904_1080 (size=12301) 2024-11-19T12:51:52,976 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/54ecbbcc43804a05a69de5e9f8bc1aa5 2024-11-19T12:51:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020772958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020772959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020772965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020772976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020772978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a76031ccf16b4a908457ea9fb0638aa7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7 2024-11-19T12:51:53,009 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T12:51:53,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/3de4d32647d64b559f49f58e1e0ff06f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f 2024-11-19T12:51:53,017 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T12:51:53,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/54ecbbcc43804a05a69de5e9f8bc1aa5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5 2024-11-19T12:51:53,032 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T12:51:53,033 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 215ms, sequenceid=313, compaction requested=true 2024-11-19T12:51:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T12:51:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T12:51:53,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T12:51:53,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-11-19T12:51:53,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 849 msec 2024-11-19T12:51:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:53,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:51:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:53,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:53,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d7119796ecb1463db856534ccba823f6 is 50, key is test_row_0/A:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741905_1081 (size=14741) 2024-11-19T12:51:53,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d7119796ecb1463db856534ccba823f6 2024-11-19T12:51:53,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a2fcd1153cf54853a83186a1ff2f1d6b is 50, key is test_row_0/B:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020773109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020773111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020773112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020773122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020773124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741906_1082 (size=12301) 2024-11-19T12:51:53,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a2fcd1153cf54853a83186a1ff2f1d6b 2024-11-19T12:51:53,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/880fea252291466caa4083c76c2cfdf1 is 50, key is test_row_0/C:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741907_1083 (size=12301) 2024-11-19T12:51:53,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/880fea252291466caa4083c76c2cfdf1 2024-11-19T12:51:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d7119796ecb1463db856534ccba823f6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6 2024-11-19T12:51:53,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6, entries=200, sequenceid=331, filesize=14.4 K 2024-11-19T12:51:53,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a2fcd1153cf54853a83186a1ff2f1d6b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b 2024-11-19T12:51:53,232 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/159cbedf951a4ee5939453232418adf5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/159cbedf951a4ee5939453232418adf5 2024-11-19T12:51:53,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T12:51:53,237 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ac37dfdf5cf141d28ec15b436822c045 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ac37dfdf5cf141d28ec15b436822c045 2024-11-19T12:51:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/880fea252291466caa4083c76c2cfdf1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1 2024-11-19T12:51:53,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020773234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020773234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020773234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,246 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 159cbedf951a4ee5939453232418adf5(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:53,246 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,246 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020712799; duration=0sec 2024-11-19T12:51:53,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T12:51:53,246 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:53,246 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:53,246 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T12:51:53,247 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into ac37dfdf5cf141d28ec15b436822c045(size=12.7 K), total size for store is 39.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:53,247 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,247 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020712799; duration=0sec 2024-11-19T12:51:53,248 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:53,248 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:53,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 86dc49f9832c693612c21abbffe191c4 in 152ms, sequenceid=331, compaction requested=true 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T12:51:53,248 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61969 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:53,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:53,249 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. because compaction request was cancelled 2024-11-19T12:51:53,249 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0e5ebc4c494c445f85454c2a65977a43, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=60.5 K 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:53,249 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:53,250 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5ebc4c494c445f85454c2a65977a43, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732020711145 2024-11-19T12:51:53,251 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae2f5cc8e684969ac4d6bedc93d5e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020711204 2024-11-19T12:51:53,251 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:53,251 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:53,251 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,251 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ac37dfdf5cf141d28ec15b436822c045, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=39.1 K 2024-11-19T12:51:53,252 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0da1c180bf4b88982d1833d34f7c27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732020711921 2024-11-19T12:51:53,252 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac37dfdf5cf141d28ec15b436822c045, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732020711921 2024-11-19T12:51:53,252 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ecbbcc43804a05a69de5e9f8bc1aa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020712287 2024-11-19T12:51:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:53,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:53,252 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76031ccf16b4a908457ea9fb0638aa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020712287 2024-11-19T12:51:53,252 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 880fea252291466caa4083c76c2cfdf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712962 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:53,253 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7119796ecb1463db856534ccba823f6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712950 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:53,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b68b123a99a44155a054d526e3d33310 is 50, key is test_row_0/A:col10/1732020713102/Put/seqid=0 2024-11-19T12:51:53,297 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:53,298 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/8de433d7c361464099728fc231a2c71f is 50, key is test_row_0/A:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,306 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:53,307 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2b01192b8ee74c3dab136d32d410ab53 is 50, key is test_row_0/C:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020773326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T12:51:53,332 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:53,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020773330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T12:51:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T12:51:53,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:53,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:53,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:53,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741908_1084 (size=14741) 2024-11-19T12:51:53,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b68b123a99a44155a054d526e3d33310 2024-11-19T12:51:53,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741909_1085 (size=13119) 2024-11-19T12:51:53,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741910_1086 (size=13085) 2024-11-19T12:51:53,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/dd08acdabef34f1e801777e98983555f is 50, key is test_row_0/B:col10/1732020713102/Put/seqid=0 2024-11-19T12:51:53,371 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2b01192b8ee74c3dab136d32d410ab53 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2b01192b8ee74c3dab136d32d410ab53 2024-11-19T12:51:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741911_1087 (size=12301) 2024-11-19T12:51:53,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/dd08acdabef34f1e801777e98983555f 2024-11-19T12:51:53,382 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 2b01192b8ee74c3dab136d32d410ab53(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:53,383 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,383 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=11, startTime=1732020712799; duration=0sec 2024-11-19T12:51:53,383 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:53,383 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:53,383 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:53,384 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:53,384 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:53,384 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,385 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/159cbedf951a4ee5939453232418adf5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=36.7 K 2024-11-19T12:51:53,385 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 159cbedf951a4ee5939453232418adf5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732020711921 2024-11-19T12:51:53,386 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de4d32647d64b559f49f58e1e0ff06f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020712287 2024-11-19T12:51:53,387 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a2fcd1153cf54853a83186a1ff2f1d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712962 2024-11-19T12:51:53,397 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:53,397 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/67382ad85ac3434d8e19a4f94f590b81 is 50, key is test_row_0/B:col10/1732020712962/Put/seqid=0 2024-11-19T12:51:53,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2304b2d0f1fc46bdb35c7b0b12e3f75f is 50, key is test_row_0/C:col10/1732020713102/Put/seqid=0 2024-11-19T12:51:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741912_1088 (size=13119) 2024-11-19T12:51:53,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741913_1089 (size=12301) 2024-11-19T12:51:53,439 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/67382ad85ac3434d8e19a4f94f590b81 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/67382ad85ac3434d8e19a4f94f590b81 2024-11-19T12:51:53,446 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 67382ad85ac3434d8e19a4f94f590b81(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:53,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,446 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020713248; duration=0sec 2024-11-19T12:51:53,447 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:53,447 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:53,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020773441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020773442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020773442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T12:51:53,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020773451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020773451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T12:51:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T12:51:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020773651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020773651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T12:51:53,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020773751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020773762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020773762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:53,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:53,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:53,766 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/8de433d7c361464099728fc231a2c71f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8de433d7c361464099728fc231a2c71f 2024-11-19T12:51:53,773 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 8de433d7c361464099728fc231a2c71f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:53,773 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,773 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020713248; duration=0sec 2024-11-19T12:51:53,773 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:53,773 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:53,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T12:51:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:53,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2304b2d0f1fc46bdb35c7b0b12e3f75f 2024-11-19T12:51:53,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/b68b123a99a44155a054d526e3d33310 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310 2024-11-19T12:51:53,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310, entries=200, sequenceid=348, filesize=14.4 K 2024-11-19T12:51:53,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/dd08acdabef34f1e801777e98983555f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f 2024-11-19T12:51:53,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f, entries=150, sequenceid=348, filesize=12.0 K 2024-11-19T12:51:53,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2304b2d0f1fc46bdb35c7b0b12e3f75f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f 2024-11-19T12:51:53,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f, entries=150, sequenceid=348, filesize=12.0 K 2024-11-19T12:51:53,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 609ms, sequenceid=348, compaction requested=false 2024-11-19T12:51:53,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:53,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T12:51:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:53,962 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:51:53,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:53,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:53,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T12:51:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/11d64cc24c024b7f905e7a0f1a116061 is 50, key is test_row_0/A:col10/1732020713297/Put/seqid=0 2024-11-19T12:51:53,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741914_1090 (size=12301) 2024-11-19T12:51:53,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/11d64cc24c024b7f905e7a0f1a116061 2024-11-19T12:51:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/605a6030a9f446808443996ec8dcdf82 is 50, key is test_row_0/B:col10/1732020713297/Put/seqid=0 2024-11-19T12:51:54,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741915_1091 (size=12301) 2024-11-19T12:51:54,010 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/605a6030a9f446808443996ec8dcdf82 2024-11-19T12:51:54,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/38d4fb35a36245378ca6f02fbfeb199e is 50, key is test_row_0/C:col10/1732020713297/Put/seqid=0 2024-11-19T12:51:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741916_1092 (size=12301) 2024-11-19T12:51:54,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020774045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020774046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020774152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020774152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020774272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020774272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020774272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020774362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020774362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,426 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/38d4fb35a36245378ca6f02fbfeb199e 2024-11-19T12:51:54,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/11d64cc24c024b7f905e7a0f1a116061 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061 2024-11-19T12:51:54,437 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T12:51:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/605a6030a9f446808443996ec8dcdf82 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82 2024-11-19T12:51:54,451 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T12:51:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/38d4fb35a36245378ca6f02fbfeb199e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e 2024-11-19T12:51:54,461 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T12:51:54,462 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 500ms, sequenceid=368, compaction requested=true 2024-11-19T12:51:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T12:51:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T12:51:54,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T12:51:54,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1190 sec 2024-11-19T12:51:54,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1350 sec 2024-11-19T12:51:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T12:51:54,482 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:54,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T12:51:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T12:51:54,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:54,484 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:54,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T12:51:54,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T12:51:54,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:54,638 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:54,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d47bb4ac6db94d34b9f9d4f1c135522c is 50, key is test_row_0/A:col10/1732020713997/Put/seqid=0 2024-11-19T12:51:54,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741917_1093 (size=12301) 2024-11-19T12:51:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:54,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020774713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020774716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T12:51:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020774831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020774831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020775041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020775041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,065 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d47bb4ac6db94d34b9f9d4f1c135522c 2024-11-19T12:51:55,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/7386dc2f053448af965a389e0d667f25 is 50, key is test_row_0/B:col10/1732020713997/Put/seqid=0 2024-11-19T12:51:55,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741918_1094 (size=12301) 2024-11-19T12:51:55,084 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/7386dc2f053448af965a389e0d667f25 2024-11-19T12:51:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/6401a0b5a6614e26b8e576e169ac8202 is 50, key is test_row_0/C:col10/1732020713997/Put/seqid=0 2024-11-19T12:51:55,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741919_1095 (size=12301) 2024-11-19T12:51:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T12:51:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020775291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020775292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020775292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,301 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:51:55,301 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,301 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:51:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020775351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020775351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,499 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/6401a0b5a6614e26b8e576e169ac8202 2024-11-19T12:51:55,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d47bb4ac6db94d34b9f9d4f1c135522c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c 2024-11-19T12:51:55,515 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c, entries=150, sequenceid=383, filesize=12.0 K 2024-11-19T12:51:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/7386dc2f053448af965a389e0d667f25 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25 2024-11-19T12:51:55,537 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25, entries=150, sequenceid=383, filesize=12.0 K 2024-11-19T12:51:55,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/6401a0b5a6614e26b8e576e169ac8202 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202 2024-11-19T12:51:55,546 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202, entries=150, sequenceid=383, filesize=12.0 K 2024-11-19T12:51:55,547 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 909ms, sequenceid=383, compaction requested=true 2024-11-19T12:51:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T12:51:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T12:51:55,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T12:51:55,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0640 sec 2024-11-19T12:51:55,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0700 sec 2024-11-19T12:51:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T12:51:55,622 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:55,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T12:51:55,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:55,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:55,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T12:51:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T12:51:55,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T12:51:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:55,782 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:55,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/70412e6be88e4958abce7fa2a75fc725 is 50, key is test_row_0/A:col10/1732020714712/Put/seqid=0 2024-11-19T12:51:55,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741920_1096 (size=12301) 2024-11-19T12:51:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:55,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020775897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020775898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T12:51:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020776002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020776011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,209 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/70412e6be88e4958abce7fa2a75fc725 2024-11-19T12:51:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020776212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/57dfc3b055234996bab97575129f8d96 is 50, key is test_row_0/B:col10/1732020714712/Put/seqid=0 2024-11-19T12:51:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020776221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741921_1097 (size=12301) 2024-11-19T12:51:56,235 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/57dfc3b055234996bab97575129f8d96 2024-11-19T12:51:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/892cbd8aef4f47ec90ce27d46fe93e59 is 50, key is test_row_0/C:col10/1732020714712/Put/seqid=0 2024-11-19T12:51:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T12:51:56,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741922_1098 (size=12301) 2024-11-19T12:51:56,265 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/892cbd8aef4f47ec90ce27d46fe93e59 2024-11-19T12:51:56,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/70412e6be88e4958abce7fa2a75fc725 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725 2024-11-19T12:51:56,280 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725, entries=150, sequenceid=400, filesize=12.0 K 2024-11-19T12:51:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/57dfc3b055234996bab97575129f8d96 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96 2024-11-19T12:51:56,292 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96, entries=150, sequenceid=400, filesize=12.0 K 2024-11-19T12:51:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/892cbd8aef4f47ec90ce27d46fe93e59 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59 2024-11-19T12:51:56,299 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59, entries=150, sequenceid=400, filesize=12.0 K 2024-11-19T12:51:56,301 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 518ms, sequenceid=400, compaction requested=true 2024-11-19T12:51:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T12:51:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T12:51:56,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T12:51:56,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 674 msec 2024-11-19T12:51:56,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 683 msec 2024-11-19T12:51:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:56,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d514973ce63e43dba9ca5746eea37ae5 is 50, key is test_row_0/A:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741923_1099 (size=14741) 2024-11-19T12:51:56,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d514973ce63e43dba9ca5746eea37ae5 2024-11-19T12:51:56,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ceacb4b8735e4552ae3c69aff938ad8c is 50, key is test_row_0/B:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741924_1100 (size=12301) 2024-11-19T12:51:56,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ceacb4b8735e4552ae3c69aff938ad8c 2024-11-19T12:51:56,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/e1461bddd14d4aa6acd3332350ee386f is 50, key is test_row_0/C:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020776571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020776576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741925_1101 (size=12301) 2024-11-19T12:51:56,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/e1461bddd14d4aa6acd3332350ee386f 2024-11-19T12:51:56,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/d514973ce63e43dba9ca5746eea37ae5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5 2024-11-19T12:51:56,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5, entries=200, sequenceid=416, filesize=14.4 K 2024-11-19T12:51:56,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ceacb4b8735e4552ae3c69aff938ad8c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c 2024-11-19T12:51:56,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T12:51:56,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/e1461bddd14d4aa6acd3332350ee386f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f 2024-11-19T12:51:56,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T12:51:56,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 86dc49f9832c693612c21abbffe191c4 in 114ms, sequenceid=416, compaction requested=true 2024-11-19T12:51:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:56,637 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T12:51:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:56,637 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T12:51:56,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:56,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:56,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:56,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:56,641 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79504 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T12:51:56,642 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:56,642 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,642 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8de433d7c361464099728fc231a2c71f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=77.6 K 2024-11-19T12:51:56,643 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8de433d7c361464099728fc231a2c71f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712962 2024-11-19T12:51:56,644 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68b123a99a44155a054d526e3d33310, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732020713102 2024-11-19T12:51:56,644 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74624 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T12:51:56,644 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:56,644 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,644 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/67382ad85ac3434d8e19a4f94f590b81, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=72.9 K 2024-11-19T12:51:56,645 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d64cc24c024b7f905e7a0f1a116061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732020713297 2024-11-19T12:51:56,645 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 67382ad85ac3434d8e19a4f94f590b81, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712962 2024-11-19T12:51:56,648 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting dd08acdabef34f1e801777e98983555f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732020713102 2024-11-19T12:51:56,648 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47bb4ac6db94d34b9f9d4f1c135522c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732020713997 2024-11-19T12:51:56,648 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 605a6030a9f446808443996ec8dcdf82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732020713297 2024-11-19T12:51:56,649 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70412e6be88e4958abce7fa2a75fc725, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732020714697 2024-11-19T12:51:56,649 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7386dc2f053448af965a389e0d667f25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732020713997 2024-11-19T12:51:56,649 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d514973ce63e43dba9ca5746eea37ae5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:56,650 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 57dfc3b055234996bab97575129f8d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732020714697 2024-11-19T12:51:56,650 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ceacb4b8735e4552ae3c69aff938ad8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:56,678 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:56,679 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/301d205c29764e4fa682681b809b4c68 is 50, key is test_row_0/B:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,685 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:56,686 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5691ce18f66c468caef2eae53dd8d9d6 is 50, key is test_row_0/A:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:56,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:56,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4fa27315c080428d8ff0d981c86ed1f0 is 50, key is test_row_0/A:col10/1732020716569/Put/seqid=0 2024-11-19T12:51:56,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741926_1102 (size=13323) 2024-11-19T12:51:56,718 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/301d205c29764e4fa682681b809b4c68 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/301d205c29764e4fa682681b809b4c68 2024-11-19T12:51:56,724 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 301d205c29764e4fa682681b809b4c68(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:56,725 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:56,725 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=10, startTime=1732020716637; duration=0sec 2024-11-19T12:51:56,725 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:56,725 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:56,725 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T12:51:56,727 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74590 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T12:51:56,727 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:56,727 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,727 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2b01192b8ee74c3dab136d32d410ab53, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=72.8 K 2024-11-19T12:51:56,728 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b01192b8ee74c3dab136d32d410ab53, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732020712962 2024-11-19T12:51:56,728 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2304b2d0f1fc46bdb35c7b0b12e3f75f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732020713102 2024-11-19T12:51:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,730 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d4fb35a36245378ca6f02fbfeb199e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732020713297 2024-11-19T12:51:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020776722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,731 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6401a0b5a6614e26b8e576e169ac8202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732020713997 2024-11-19T12:51:56,732 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 892cbd8aef4f47ec90ce27d46fe93e59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732020714697 2024-11-19T12:51:56,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020776731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,735 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e1461bddd14d4aa6acd3332350ee386f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:56,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741927_1103 (size=13323) 2024-11-19T12:51:56,745 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5691ce18f66c468caef2eae53dd8d9d6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5691ce18f66c468caef2eae53dd8d9d6 2024-11-19T12:51:56,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741928_1104 (size=14741) 2024-11-19T12:51:56,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4fa27315c080428d8ff0d981c86ed1f0 2024-11-19T12:51:56,759 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 5691ce18f66c468caef2eae53dd8d9d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:56,759 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:56,759 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=10, startTime=1732020716637; duration=0sec 2024-11-19T12:51:56,759 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:56,759 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T12:51:56,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:51:56,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:51:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T12:51:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:56,765 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:51:56,766 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:51:56,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:51:56,769 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#92 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:56,769 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/faf16ce304f9451f9c882804c65bbdad is 50, key is test_row_0/C:col10/1732020715874/Put/seqid=0 2024-11-19T12:51:56,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a8bd99a1d07c4891a7c983f52b581f88 is 50, key is test_row_0/B:col10/1732020716569/Put/seqid=0 2024-11-19T12:51:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741929_1105 (size=13289) 2024-11-19T12:51:56,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741930_1106 (size=12301) 2024-11-19T12:51:56,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a8bd99a1d07c4891a7c983f52b581f88 2024-11-19T12:51:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020776842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020776842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:56,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/7c18c85463db45dfbe0cd69e6db684a0 is 50, key is test_row_0/C:col10/1732020716569/Put/seqid=0 2024-11-19T12:51:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741931_1107 (size=12301) 2024-11-19T12:51:56,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:56,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020777052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020777052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:57,226 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/faf16ce304f9451f9c882804c65bbdad as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/faf16ce304f9451f9c882804c65bbdad 2024-11-19T12:51:57,232 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into faf16ce304f9451f9c882804c65bbdad(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:57,232 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:57,232 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=10, startTime=1732020716638; duration=0sec 2024-11-19T12:51:57,233 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:57,233 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:57,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/7c18c85463db45dfbe0cd69e6db684a0 2024-11-19T12:51:57,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/4fa27315c080428d8ff0d981c86ed1f0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0 2024-11-19T12:51:57,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0, entries=200, sequenceid=433, filesize=14.4 K 2024-11-19T12:51:57,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a8bd99a1d07c4891a7c983f52b581f88 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88 2024-11-19T12:51:57,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T12:51:57,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/7c18c85463db45dfbe0cd69e6db684a0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0 2024-11-19T12:51:57,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T12:51:57,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 633ms, sequenceid=433, compaction requested=false 2024-11-19T12:51:57,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:57,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:57,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:57,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ebabf94236ee43849951aeb1e979d0cd is 50, key is test_row_0/A:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:57,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741932_1108 (size=12301) 2024-11-19T12:51:57,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ebabf94236ee43849951aeb1e979d0cd 2024-11-19T12:51:57,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020777360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020777361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020777362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020777362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020777365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ea4517fc5f254a68b8c05eaa0526131c is 50, key is test_row_0/B:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:57,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741933_1109 (size=12301) 2024-11-19T12:51:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020777472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020777472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020777481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020777681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020777681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020777691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ea4517fc5f254a68b8c05eaa0526131c 2024-11-19T12:51:57,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/5d1712b6f7704b578b598d3790add46a is 50, key is test_row_0/C:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741934_1110 (size=12301) 2024-11-19T12:51:57,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:57,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020777871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020777881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020777991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020777991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020778002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:51:58,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/5d1712b6f7704b578b598d3790add46a 2024-11-19T12:51:58,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/ebabf94236ee43849951aeb1e979d0cd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd 2024-11-19T12:51:58,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T12:51:58,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/ea4517fc5f254a68b8c05eaa0526131c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c 2024-11-19T12:51:58,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T12:51:58,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/5d1712b6f7704b578b598d3790add46a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a 2024-11-19T12:51:58,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T12:51:58,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 86dc49f9832c693612c21abbffe191c4 in 928ms, sequenceid=452, compaction requested=true 2024-11-19T12:51:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:58,253 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:51:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:58,253 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:51:58,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:58,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:51:58,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:58,256 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:58,256 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:51:58,256 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,256 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5691ce18f66c468caef2eae53dd8d9d6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=39.4 K 2024-11-19T12:51:58,257 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:58,257 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:51:58,257 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,257 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/301d205c29764e4fa682681b809b4c68, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.0 K 2024-11-19T12:51:58,258 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5691ce18f66c468caef2eae53dd8d9d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:58,258 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 301d205c29764e4fa682681b809b4c68, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:58,259 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa27315c080428d8ff0d981c86ed1f0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732020716558 2024-11-19T12:51:58,259 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a8bd99a1d07c4891a7c983f52b581f88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732020716558 2024-11-19T12:51:58,259 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebabf94236ee43849951aeb1e979d0cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:51:58,260 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4517fc5f254a68b8c05eaa0526131c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:51:58,268 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:58,268 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/95a36cedd802403298bdfb334329fa75 is 50, key is test_row_0/B:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:58,268 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:58,269 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0b3d5c1321944fb0bc45deaf47c53181 is 50, key is test_row_0/A:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:58,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741936_1112 (size=13425) 2024-11-19T12:51:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741935_1111 (size=13425) 2024-11-19T12:51:58,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,312 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:58,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/10b2177aa8f04787bcba5f36f47f1345 is 50, key is test_row_0/A:col10/1732020717363/Put/seqid=0 2024-11-19T12:51:58,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741937_1113 (size=12301) 2024-11-19T12:51:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:58,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:51:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020778526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020778529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020778529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020778631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020778642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020778642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,685 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0b3d5c1321944fb0bc45deaf47c53181 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0b3d5c1321944fb0bc45deaf47c53181 2024-11-19T12:51:58,686 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/95a36cedd802403298bdfb334329fa75 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/95a36cedd802403298bdfb334329fa75 2024-11-19T12:51:58,695 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 0b3d5c1321944fb0bc45deaf47c53181(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:58,695 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:58,695 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020718253; duration=0sec 2024-11-19T12:51:58,695 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:51:58,695 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:51:58,696 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:51:58,696 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 95a36cedd802403298bdfb334329fa75(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:58,696 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:58,696 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020718253; duration=0sec 2024-11-19T12:51:58,696 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:58,696 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:51:58,700 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:51:58,700 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:51:58,700 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:58,700 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/faf16ce304f9451f9c882804c65bbdad, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.0 K 2024-11-19T12:51:58,701 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf16ce304f9451f9c882804c65bbdad, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732020715874 2024-11-19T12:51:58,701 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c18c85463db45dfbe0cd69e6db684a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732020716558 2024-11-19T12:51:58,702 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1712b6f7704b578b598d3790add46a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:51:58,712 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:51:58,713 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/84243f473844430ea0cd0e752d8b359d is 50, key is test_row_0/C:col10/1732020716708/Put/seqid=0 2024-11-19T12:51:58,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741938_1114 (size=13391) 2024-11-19T12:51:58,729 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/10b2177aa8f04787bcba5f36f47f1345 2024-11-19T12:51:58,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/cb439ca6824c478999051b75c787a373 is 50, key is test_row_0/B:col10/1732020717363/Put/seqid=0 2024-11-19T12:51:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741939_1115 (size=12301) 2024-11-19T12:51:58,755 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/cb439ca6824c478999051b75c787a373 2024-11-19T12:51:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/72e266db696c43b2bacbcd2235318106 is 50, key is test_row_0/C:col10/1732020717363/Put/seqid=0 2024-11-19T12:51:58,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741940_1116 (size=12301) 2024-11-19T12:51:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020778841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020778851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020778851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020778881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,884 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:51:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020778891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:58,893 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:51:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:51:59,124 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/84243f473844430ea0cd0e752d8b359d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/84243f473844430ea0cd0e752d8b359d 2024-11-19T12:51:59,134 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 84243f473844430ea0cd0e752d8b359d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:51:59,134 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:59,135 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=13, startTime=1732020718254; duration=0sec 2024-11-19T12:51:59,135 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:51:59,135 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:51:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020779152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020779161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020779161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,180 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/72e266db696c43b2bacbcd2235318106 2024-11-19T12:51:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/10b2177aa8f04787bcba5f36f47f1345 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345 2024-11-19T12:51:59,193 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T12:51:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/cb439ca6824c478999051b75c787a373 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373 2024-11-19T12:51:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T12:51:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/72e266db696c43b2bacbcd2235318106 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106 2024-11-19T12:51:59,212 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T12:51:59,213 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 86dc49f9832c693612c21abbffe191c4 in 901ms, sequenceid=468, compaction requested=false 2024-11-19T12:51:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:51:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:51:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T12:51:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T12:51:59,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T12:51:59,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4480 sec 2024-11-19T12:51:59,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.4530 sec 2024-11-19T12:51:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:51:59,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:51:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:51:59,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/7a5d89274fed498a88d40ccb23086178 is 50, key is test_row_0/A:col10/1732020718527/Put/seqid=0 2024-11-19T12:51:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741941_1117 (size=12301) 2024-11-19T12:51:59,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020779689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020779694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020779695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020779801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020779811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:51:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:51:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020779811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:51:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:51:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020780011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020780021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020780021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/7a5d89274fed498a88d40ccb23086178 2024-11-19T12:52:00,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/152eb2604e8f477ab04834bed64a2ce5 is 50, key is test_row_0/B:col10/1732020718527/Put/seqid=0 2024-11-19T12:52:00,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741942_1118 (size=12301) 2024-11-19T12:52:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020780322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020780331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020780331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/152eb2604e8f477ab04834bed64a2ce5 2024-11-19T12:52:00,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2a294db93118424482192e06b68b6cb1 is 50, key is test_row_0/C:col10/1732020718527/Put/seqid=0 2024-11-19T12:52:00,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741943_1119 (size=12301) 2024-11-19T12:52:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46126 deadline: 1732020780831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46182 deadline: 1732020780841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1732020780841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46128 deadline: 1732020780901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,904 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2a294db93118424482192e06b68b6cb1 2024-11-19T12:52:00,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732020780912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:00,914 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 86dc49f9832c693612c21abbffe191c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/7a5d89274fed498a88d40ccb23086178 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178 2024-11-19T12:52:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:00,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178, entries=150, sequenceid=489, filesize=12.0 K 2024-11-19T12:52:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T12:52:00,921 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:00,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T12:52:00,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/152eb2604e8f477ab04834bed64a2ce5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5 2024-11-19T12:52:00,924 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:00,925 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:00,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:00,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5, entries=150, sequenceid=489, filesize=12.0 K 2024-11-19T12:52:00,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/2a294db93118424482192e06b68b6cb1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1 2024-11-19T12:52:00,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1, entries=150, sequenceid=489, filesize=12.0 K 2024-11-19T12:52:00,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 86dc49f9832c693612c21abbffe191c4 in 1274ms, sequenceid=489, compaction requested=true 2024-11-19T12:52:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:52:00,937 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:00,938 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86dc49f9832c693612c21abbffe191c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:00,938 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:00,939 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/A is initiating minor compaction (all files) 2024-11-19T12:52:00,939 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/A in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:00,939 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0b3d5c1321944fb0bc45deaf47c53181, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.1 K 2024-11-19T12:52:00,940 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:00,940 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/B is initiating minor compaction (all files) 2024-11-19T12:52:00,940 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/B in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:00,940 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/95a36cedd802403298bdfb334329fa75, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.1 K 2024-11-19T12:52:00,941 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3d5c1321944fb0bc45deaf47c53181, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:52:00,941 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a36cedd802403298bdfb334329fa75, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:52:00,941 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b2177aa8f04787bcba5f36f47f1345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732020717355 2024-11-19T12:52:00,942 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting cb439ca6824c478999051b75c787a373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732020717355 2024-11-19T12:52:00,942 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5d89274fed498a88d40ccb23086178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732020718527 2024-11-19T12:52:00,943 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 152eb2604e8f477ab04834bed64a2ce5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732020718527 2024-11-19T12:52:00,953 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#A#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:00,953 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0c3b116a7aa64ceabc650e8cb88546dd is 50, key is test_row_0/A:col10/1732020718527/Put/seqid=0 2024-11-19T12:52:00,958 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:00,958 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/9792973c288d445c9c68405c1ab560b5 is 50, key is test_row_0/B:col10/1732020718527/Put/seqid=0 2024-11-19T12:52:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741945_1121 (size=13527) 2024-11-19T12:52:00,986 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/9792973c288d445c9c68405c1ab560b5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/9792973c288d445c9c68405c1ab560b5 2024-11-19T12:52:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741944_1120 (size=13527) 2024-11-19T12:52:00,993 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/0c3b116a7aa64ceabc650e8cb88546dd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0c3b116a7aa64ceabc650e8cb88546dd 2024-11-19T12:52:00,994 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/B of 86dc49f9832c693612c21abbffe191c4 into 9792973c288d445c9c68405c1ab560b5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:00,994 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:52:00,995 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/B, priority=13, startTime=1732020720938; duration=0sec 2024-11-19T12:52:00,998 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:00,998 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:B 2024-11-19T12:52:00,998 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:01,003 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:01,004 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 86dc49f9832c693612c21abbffe191c4/C is initiating minor compaction (all files) 2024-11-19T12:52:01,004 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86dc49f9832c693612c21abbffe191c4/C in TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:01,004 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/84243f473844430ea0cd0e752d8b359d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp, totalSize=37.1 K 2024-11-19T12:52:01,005 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/A of 86dc49f9832c693612c21abbffe191c4 into 0c3b116a7aa64ceabc650e8cb88546dd(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:01,005 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:52:01,005 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/A, priority=13, startTime=1732020720937; duration=0sec 2024-11-19T12:52:01,005 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 84243f473844430ea0cd0e752d8b359d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732020716708 2024-11-19T12:52:01,005 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:01,005 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:A 2024-11-19T12:52:01,008 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e266db696c43b2bacbcd2235318106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1732020717355 2024-11-19T12:52:01,009 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a294db93118424482192e06b68b6cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732020718527 2024-11-19T12:52:01,020 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86dc49f9832c693612c21abbffe191c4#C#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:01,021 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0585f7004e86406a9efb7d43f0d040de is 50, key is test_row_0/C:col10/1732020718527/Put/seqid=0 2024-11-19T12:52:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741946_1122 (size=13493) 2024-11-19T12:52:01,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T12:52:01,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:01,078 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:01,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:52:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:52:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:52:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a1e18c4b77614d938191954b4dfcc37f is 50, key is test_row_0/A:col10/1732020719666/Put/seqid=0 2024-11-19T12:52:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741947_1123 (size=12301) 2024-11-19T12:52:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:01,442 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/0585f7004e86406a9efb7d43f0d040de as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0585f7004e86406a9efb7d43f0d040de 2024-11-19T12:52:01,447 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86dc49f9832c693612c21abbffe191c4/C of 86dc49f9832c693612c21abbffe191c4 into 0585f7004e86406a9efb7d43f0d040de(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:01,448 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:52:01,448 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., storeName=86dc49f9832c693612c21abbffe191c4/C, priority=13, startTime=1732020720938; duration=0sec 2024-11-19T12:52:01,448 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:01,448 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86dc49f9832c693612c21abbffe191c4:C 2024-11-19T12:52:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a1e18c4b77614d938191954b4dfcc37f 2024-11-19T12:52:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/93bb61c7029b49459f2893548da69817 is 50, key is test_row_0/B:col10/1732020719666/Put/seqid=0 2024-11-19T12:52:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741948_1124 (size=12301) 2024-11-19T12:52:01,533 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T12:52:01,533 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T12:52:01,533 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,533 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,534 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T12:52:01,534 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,537 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T12:52:01,537 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,537 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,537 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,537 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,537 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,538 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,538 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,538 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,538 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:52:01,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. as already flushing 2024-11-19T12:52:01,843 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T12:52:01,843 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,843 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,843 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,853 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T12:52:01,853 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,853 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,853 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,864 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T12:52:01,864 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:01,864 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,864 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:01,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:01,902 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/93bb61c7029b49459f2893548da69817 2024-11-19T12:52:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/42cfe581a37e46c9869a20f90c31616c is 50, key is test_row_0/C:col10/1732020719666/Put/seqid=0 2024-11-19T12:52:01,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741949_1125 (size=12301) 2024-11-19T12:52:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:02,316 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/42cfe581a37e46c9869a20f90c31616c 2024-11-19T12:52:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/a1e18c4b77614d938191954b4dfcc37f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a1e18c4b77614d938191954b4dfcc37f 2024-11-19T12:52:02,327 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a1e18c4b77614d938191954b4dfcc37f, entries=150, sequenceid=506, filesize=12.0 K 2024-11-19T12:52:02,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/93bb61c7029b49459f2893548da69817 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/93bb61c7029b49459f2893548da69817 2024-11-19T12:52:02,332 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/93bb61c7029b49459f2893548da69817, entries=150, sequenceid=506, filesize=12.0 K 2024-11-19T12:52:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/42cfe581a37e46c9869a20f90c31616c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/42cfe581a37e46c9869a20f90c31616c 2024-11-19T12:52:02,338 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/42cfe581a37e46c9869a20f90c31616c, entries=150, sequenceid=506, filesize=12.0 K 2024-11-19T12:52:02,338 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 86dc49f9832c693612c21abbffe191c4 in 1260ms, sequenceid=506, compaction requested=false 2024-11-19T12:52:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 86dc49f9832c693612c21abbffe191c4: 2024-11-19T12:52:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T12:52:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T12:52:02,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T12:52:02,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-19T12:52:02,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4200 sec 2024-11-19T12:52:02,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:52:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T12:52:03,071 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:04,913 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T12:52:04,913 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:04,913 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,913 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:04,933 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T12:52:04,933 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:04,933 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,933 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:04,933 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2826 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2650 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1197 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3587 rows 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1203 2024-11-19T12:52:04,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3605 rows 2024-11-19T12:52:04,934 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:52:04,935 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:52:04,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:04,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:04,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:52:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:04,946 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020724946"}]},"ts":"1732020724946"} 2024-11-19T12:52:04,948 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:52:04,948 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:52:04,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:52:04,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, UNASSIGN}] 2024-11-19T12:52:04,955 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, UNASSIGN 2024-11-19T12:52:04,956 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=86dc49f9832c693612c21abbffe191c4, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:04,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, UNASSIGN because future has completed 2024-11-19T12:52:04,958 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:52:04,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:05,114 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:52:05,115 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:52:05,115 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 86dc49f9832c693612c21abbffe191c4, disabling compactions & flushes 2024-11-19T12:52:05,115 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:05,115 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:05,115 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. after waiting 0 ms 2024-11-19T12:52:05,115 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:05,115 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 86dc49f9832c693612c21abbffe191c4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=A 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=B 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86dc49f9832c693612c21abbffe191c4, store=C 2024-11-19T12:52:05,116 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:05,120 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5808121743ab414782b6e8573c9fc787 is 50, key is test_row_0/A:col10/1732020724911/Put/seqid=0 2024-11-19T12:52:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741950_1126 (size=12301) 2024-11-19T12:52:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:05,525 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5808121743ab414782b6e8573c9fc787 2024-11-19T12:52:05,537 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a799bb1bd4e54da8860b2508e215bb10 is 50, key is test_row_0/B:col10/1732020724911/Put/seqid=0 2024-11-19T12:52:05,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741951_1127 (size=12301) 2024-11-19T12:52:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:05,944 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a799bb1bd4e54da8860b2508e215bb10 2024-11-19T12:52:05,958 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/32cd6926fbff40c3b2c8dbd738a973b0 is 50, key is test_row_0/C:col10/1732020724911/Put/seqid=0 2024-11-19T12:52:05,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741952_1128 (size=12301) 2024-11-19T12:52:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:06,364 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/32cd6926fbff40c3b2c8dbd738a973b0 2024-11-19T12:52:06,376 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/A/5808121743ab414782b6e8573c9fc787 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5808121743ab414782b6e8573c9fc787 2024-11-19T12:52:06,385 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5808121743ab414782b6e8573c9fc787, entries=150, sequenceid=515, filesize=12.0 K 2024-11-19T12:52:06,387 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/B/a799bb1bd4e54da8860b2508e215bb10 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a799bb1bd4e54da8860b2508e215bb10 2024-11-19T12:52:06,394 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a799bb1bd4e54da8860b2508e215bb10, entries=150, sequenceid=515, filesize=12.0 K 2024-11-19T12:52:06,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/.tmp/C/32cd6926fbff40c3b2c8dbd738a973b0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/32cd6926fbff40c3b2c8dbd738a973b0 2024-11-19T12:52:06,400 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/32cd6926fbff40c3b2c8dbd738a973b0, entries=150, sequenceid=515, filesize=12.0 K 2024-11-19T12:52:06,401 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 86dc49f9832c693612c21abbffe191c4 in 1286ms, sequenceid=515, compaction requested=true 2024-11-19T12:52:06,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5741fb16a2cc433fa383bf8fef9d6a72, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/2083ca8bc1ac4bf084d9b56eb910f2cc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a92f7625c93842a9a186a91f56e4f614, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/45f3159613ac49f1adea7287306b3c1f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/91c80e259ce340ef928ec09c66b3e22c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ac37dfdf5cf141d28ec15b436822c045, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8de433d7c361464099728fc231a2c71f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5691ce18f66c468caef2eae53dd8d9d6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0b3d5c1321944fb0bc45deaf47c53181, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178] to archive 2024-11-19T12:52:06,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:06,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/75f58ec12efe41b38dcc732f4cace691 2024-11-19T12:52:06,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ae83ba86f81c4c03baa59e990796a530 2024-11-19T12:52:06,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5741fb16a2cc433fa383bf8fef9d6a72 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5741fb16a2cc433fa383bf8fef9d6a72 2024-11-19T12:52:06,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/940c99ae6aef4d069e70a424e2faaab6 2024-11-19T12:52:06,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/f5c77cc538114f06b93b14576d1371b8 2024-11-19T12:52:06,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0e2e3b9d71d24fb68c85ad16ad62e45f 2024-11-19T12:52:06,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/2083ca8bc1ac4bf084d9b56eb910f2cc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/2083ca8bc1ac4bf084d9b56eb910f2cc 2024-11-19T12:52:06,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/68085e82d1284f798cf94fee3e44c7bc 2024-11-19T12:52:06,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/dcbb03e4aa5f459b88a83140ad35a8e8 2024-11-19T12:52:06,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4126943d846b445ea8255f41f6ae4df3 2024-11-19T12:52:06,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/088055cee7cd41b6b33a03120834fb41 2024-11-19T12:52:06,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a92f7625c93842a9a186a91f56e4f614 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a92f7625c93842a9a186a91f56e4f614 2024-11-19T12:52:06,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5fac15c438b3459893ad5d9fbbcd8bb6 2024-11-19T12:52:06,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8ce1c3a9147d487797d61c0ea9420de8 2024-11-19T12:52:06,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/45f3159613ac49f1adea7287306b3c1f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/45f3159613ac49f1adea7287306b3c1f 2024-11-19T12:52:06,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/1ce7a36e1bd04ce99def07a5c93fc80c 2024-11-19T12:52:06,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0170b78980694126aa3f1536a9f25a19 2024-11-19T12:52:06,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/91c80e259ce340ef928ec09c66b3e22c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/91c80e259ce340ef928ec09c66b3e22c 2024-11-19T12:52:06,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b6b4baf41eac4995a737b785cc1174ca 2024-11-19T12:52:06,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/6e92f4ee6b594390a3bcd2994fc720c7 2024-11-19T12:52:06,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ac37dfdf5cf141d28ec15b436822c045 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ac37dfdf5cf141d28ec15b436822c045 2024-11-19T12:52:06,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0f5cfa31292546e59b5858f208869a96 2024-11-19T12:52:06,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a76031ccf16b4a908457ea9fb0638aa7 2024-11-19T12:52:06,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d7119796ecb1463db856534ccba823f6 2024-11-19T12:52:06,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8de433d7c361464099728fc231a2c71f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/8de433d7c361464099728fc231a2c71f 2024-11-19T12:52:06,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/b68b123a99a44155a054d526e3d33310 2024-11-19T12:52:06,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/11d64cc24c024b7f905e7a0f1a116061 2024-11-19T12:52:06,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d47bb4ac6db94d34b9f9d4f1c135522c 2024-11-19T12:52:06,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/70412e6be88e4958abce7fa2a75fc725 2024-11-19T12:52:06,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/d514973ce63e43dba9ca5746eea37ae5 2024-11-19T12:52:06,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5691ce18f66c468caef2eae53dd8d9d6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5691ce18f66c468caef2eae53dd8d9d6 2024-11-19T12:52:06,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/4fa27315c080428d8ff0d981c86ed1f0 2024-11-19T12:52:06,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0b3d5c1321944fb0bc45deaf47c53181 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0b3d5c1321944fb0bc45deaf47c53181 2024-11-19T12:52:06,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/ebabf94236ee43849951aeb1e979d0cd 2024-11-19T12:52:06,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/10b2177aa8f04787bcba5f36f47f1345 2024-11-19T12:52:06,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/7a5d89274fed498a88d40ccb23086178 2024-11-19T12:52:06,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/742dc05c24b6406397467a6f20f5acf2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/96e0479bee514b3e8147b56b5bd9bf8b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a73d1fae66a14ba9841aa917b543207f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/edf99242c3cc4a36a704880a55bc7253, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d0e2c1e80fd841b5b005049a4602ee4d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/159cbedf951a4ee5939453232418adf5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/67382ad85ac3434d8e19a4f94f590b81, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/301d205c29764e4fa682681b809b4c68, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/95a36cedd802403298bdfb334329fa75, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5] to archive 2024-11-19T12:52:06,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:06,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/40df6a6e9d5f491d809d698bab04a648 2024-11-19T12:52:06,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ad61dd1a2a42459eae1736cba356025a 2024-11-19T12:52:06,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/742dc05c24b6406397467a6f20f5acf2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/742dc05c24b6406397467a6f20f5acf2 2024-11-19T12:52:06,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/58009f3141fc433f9484e81d03b30ac4 2024-11-19T12:52:06,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/18346b74e6114493aa1dc925ad27f53d 2024-11-19T12:52:06,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ce18b15e17b0483e99079e5f8e064f8c 2024-11-19T12:52:06,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/96e0479bee514b3e8147b56b5bd9bf8b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/96e0479bee514b3e8147b56b5bd9bf8b 2024-11-19T12:52:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6770191535bd41b8952797ae007d314d 2024-11-19T12:52:06,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/0ec08b2e3df64846b388e3fe0aa095c2 2024-11-19T12:52:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/c6cb15291b6f4dc48932767f5fe13998 2024-11-19T12:52:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a73d1fae66a14ba9841aa917b543207f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a73d1fae66a14ba9841aa917b543207f 2024-11-19T12:52:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d58a70e06ae34350a0c0f12fce147321 2024-11-19T12:52:06,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/6c1694f2e681401e9c83729f5d6251eb 2024-11-19T12:52:06,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/edf99242c3cc4a36a704880a55bc7253 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/edf99242c3cc4a36a704880a55bc7253 2024-11-19T12:52:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d8e90366615047b39ec1a9f9bf29af42 2024-11-19T12:52:06,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ed315bb508d14dcba3b176f354ce39ee 2024-11-19T12:52:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/267a4956449d42b4b67f6946c10c057b 2024-11-19T12:52:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d0e2c1e80fd841b5b005049a4602ee4d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/d0e2c1e80fd841b5b005049a4602ee4d 2024-11-19T12:52:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/24e597d0b3f64faea3e31c3050e367bd 2024-11-19T12:52:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/4c4fd9f1dd414011ae9be45b26333f90 2024-11-19T12:52:06,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/159cbedf951a4ee5939453232418adf5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/159cbedf951a4ee5939453232418adf5 2024-11-19T12:52:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/8d99e7cb77fb40fe82413e6016285efb 2024-11-19T12:52:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/3de4d32647d64b559f49f58e1e0ff06f 2024-11-19T12:52:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/67382ad85ac3434d8e19a4f94f590b81 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/67382ad85ac3434d8e19a4f94f590b81 2024-11-19T12:52:06,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a2fcd1153cf54853a83186a1ff2f1d6b 2024-11-19T12:52:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/dd08acdabef34f1e801777e98983555f 2024-11-19T12:52:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/605a6030a9f446808443996ec8dcdf82 2024-11-19T12:52:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/7386dc2f053448af965a389e0d667f25 2024-11-19T12:52:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/57dfc3b055234996bab97575129f8d96 2024-11-19T12:52:06,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/301d205c29764e4fa682681b809b4c68 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/301d205c29764e4fa682681b809b4c68 2024-11-19T12:52:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ceacb4b8735e4552ae3c69aff938ad8c 2024-11-19T12:52:06,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a8bd99a1d07c4891a7c983f52b581f88 2024-11-19T12:52:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/95a36cedd802403298bdfb334329fa75 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/95a36cedd802403298bdfb334329fa75 2024-11-19T12:52:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/ea4517fc5f254a68b8c05eaa0526131c 2024-11-19T12:52:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/cb439ca6824c478999051b75c787a373 2024-11-19T12:52:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/152eb2604e8f477ab04834bed64a2ce5 2024-11-19T12:52:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/abc754d54e474046b8806beea2bc1c14, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/3030ac1ba29f4f51a43de7579db490e3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a00ec95e405748dcada04d71485170f2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/511f5e1bd280415ab83c48f11018ce70, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0e5ebc4c494c445f85454c2a65977a43, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2b01192b8ee74c3dab136d32d410ab53, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/faf16ce304f9451f9c882804c65bbdad, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/84243f473844430ea0cd0e752d8b359d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1] to archive 2024-11-19T12:52:06,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/799e265efbc9471f869cf76613a97d17 2024-11-19T12:52:06,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4825cc0c3ede42f6ab9eb5c90fa8cc82 2024-11-19T12:52:06,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/abc754d54e474046b8806beea2bc1c14 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/abc754d54e474046b8806beea2bc1c14 2024-11-19T12:52:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/1798cca2e4e04b5ea699a1d57e6acade 2024-11-19T12:52:06,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0daa22f0c0e3419b9b7cefd3c973710f 2024-11-19T12:52:06,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/4e82e14d25834cfcafa372b655841140 2024-11-19T12:52:06,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/3030ac1ba29f4f51a43de7579db490e3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/3030ac1ba29f4f51a43de7579db490e3 2024-11-19T12:52:06,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/77d88f0ed89b452b974d339b48909d67 2024-11-19T12:52:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/81cc21da24b443ffbe9a8c87e7f28850 2024-11-19T12:52:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/c483fdeda3ef4b6ea154370521e23c4c 2024-11-19T12:52:06,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a00ec95e405748dcada04d71485170f2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a00ec95e405748dcada04d71485170f2 2024-11-19T12:52:06,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/df8aeed9ce5c4c459e9daf0f6529ba15 2024-11-19T12:52:06,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bb31f9cc42bd4f6baf1ed0ade375a212 2024-11-19T12:52:06,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/511f5e1bd280415ab83c48f11018ce70 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/511f5e1bd280415ab83c48f11018ce70 2024-11-19T12:52:06,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/bdf1e654c28f4a84b7a85f30f0be1615 2024-11-19T12:52:06,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/83f837abd6844406a95ebd18d66832e7 2024-11-19T12:52:06,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/85808c53cb7246608220e19b97d970e6 2024-11-19T12:52:06,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0e5ebc4c494c445f85454c2a65977a43 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0e5ebc4c494c445f85454c2a65977a43 2024-11-19T12:52:06,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/a41b04ea5e444fb4b42458bb796dc191 2024-11-19T12:52:06,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2ae2f5cc8e684969ac4d6bedc93d5e0d 2024-11-19T12:52:06,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0d0da1c180bf4b88982d1833d34f7c27 2024-11-19T12:52:06,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/54ecbbcc43804a05a69de5e9f8bc1aa5 2024-11-19T12:52:06,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2b01192b8ee74c3dab136d32d410ab53 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2b01192b8ee74c3dab136d32d410ab53 2024-11-19T12:52:06,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/880fea252291466caa4083c76c2cfdf1 2024-11-19T12:52:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2304b2d0f1fc46bdb35c7b0b12e3f75f 2024-11-19T12:52:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/38d4fb35a36245378ca6f02fbfeb199e 2024-11-19T12:52:06,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/6401a0b5a6614e26b8e576e169ac8202 2024-11-19T12:52:06,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/892cbd8aef4f47ec90ce27d46fe93e59 2024-11-19T12:52:06,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/faf16ce304f9451f9c882804c65bbdad to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/faf16ce304f9451f9c882804c65bbdad 2024-11-19T12:52:06,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/e1461bddd14d4aa6acd3332350ee386f 2024-11-19T12:52:06,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/7c18c85463db45dfbe0cd69e6db684a0 2024-11-19T12:52:06,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/84243f473844430ea0cd0e752d8b359d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/84243f473844430ea0cd0e752d8b359d 2024-11-19T12:52:06,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/5d1712b6f7704b578b598d3790add46a 2024-11-19T12:52:06,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/72e266db696c43b2bacbcd2235318106 2024-11-19T12:52:06,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/2a294db93118424482192e06b68b6cb1 2024-11-19T12:52:06,555 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/recovered.edits/518.seqid, newMaxSeqId=518, maxSeqId=1 2024-11-19T12:52:06,556 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. 2024-11-19T12:52:06,556 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 86dc49f9832c693612c21abbffe191c4: Waiting for close lock at 1732020725115Running coprocessor pre-close hooks at 1732020725115Disabling compacts and flushes for region at 1732020725115Disabling writes for close at 1732020725115Obtaining lock to block concurrent updates at 1732020725116 (+1 ms)Preparing flush snapshotting stores in 86dc49f9832c693612c21abbffe191c4 at 1732020725116Finished memstore snapshotting TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732020725116Flushing stores of TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4. at 1732020725117 (+1 ms)Flushing 86dc49f9832c693612c21abbffe191c4/A: creating writer at 1732020725117Flushing 86dc49f9832c693612c21abbffe191c4/A: appending metadata at 1732020725120 (+3 ms)Flushing 86dc49f9832c693612c21abbffe191c4/A: closing flushed file at 1732020725120Flushing 86dc49f9832c693612c21abbffe191c4/B: creating writer at 1732020725533 (+413 ms)Flushing 86dc49f9832c693612c21abbffe191c4/B: appending metadata at 1732020725537 (+4 ms)Flushing 86dc49f9832c693612c21abbffe191c4/B: closing flushed file at 1732020725537Flushing 86dc49f9832c693612c21abbffe191c4/C: creating writer at 1732020725955 (+418 ms)Flushing 86dc49f9832c693612c21abbffe191c4/C: appending metadata at 1732020725958 (+3 ms)Flushing 86dc49f9832c693612c21abbffe191c4/C: closing flushed file at 1732020725958Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44fbdc9: reopening flushed file at 1732020726374 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f8dfe71: reopening flushed file at 1732020726386 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6960b62d: reopening flushed file at 1732020726394 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 86dc49f9832c693612c21abbffe191c4 in 1286ms, sequenceid=515, compaction requested=true at 1732020726401 (+7 ms)Writing region close event to WAL at 1732020726552 (+151 ms)Running coprocessor post-close hooks at 1732020726556 (+4 ms)Closed at 1732020726556 2024-11-19T12:52:06,558 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 86dc49f9832c693612c21abbffe191c4 2024-11-19T12:52:06,559 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=86dc49f9832c693612c21abbffe191c4, regionState=CLOSED 2024-11-19T12:52:06,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:06,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T12:52:06,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 86dc49f9832c693612c21abbffe191c4, server=aba5a916dfea,35349,1732020694570 in 1.6020 sec 2024-11-19T12:52:06,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T12:52:06,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86dc49f9832c693612c21abbffe191c4, UNASSIGN in 1.6090 sec 2024-11-19T12:52:06,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T12:52:06,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6160 sec 2024-11-19T12:52:06,568 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020726568"}]},"ts":"1732020726568"} 2024-11-19T12:52:06,570 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:52:06,570 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:52:06,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6310 sec 2024-11-19T12:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T12:52:07,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:07,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:52:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,107 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T12:52:07,108 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,116 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4 2024-11-19T12:52:07,121 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/recovered.edits] 2024-11-19T12:52:07,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0c3b116a7aa64ceabc650e8cb88546dd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/0c3b116a7aa64ceabc650e8cb88546dd 2024-11-19T12:52:07,128 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5808121743ab414782b6e8573c9fc787 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/5808121743ab414782b6e8573c9fc787 2024-11-19T12:52:07,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a1e18c4b77614d938191954b4dfcc37f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/A/a1e18c4b77614d938191954b4dfcc37f 2024-11-19T12:52:07,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/93bb61c7029b49459f2893548da69817 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/93bb61c7029b49459f2893548da69817 2024-11-19T12:52:07,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/9792973c288d445c9c68405c1ab560b5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/9792973c288d445c9c68405c1ab560b5 2024-11-19T12:52:07,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a799bb1bd4e54da8860b2508e215bb10 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/B/a799bb1bd4e54da8860b2508e215bb10 2024-11-19T12:52:07,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0585f7004e86406a9efb7d43f0d040de to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/0585f7004e86406a9efb7d43f0d040de 2024-11-19T12:52:07,138 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/32cd6926fbff40c3b2c8dbd738a973b0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/32cd6926fbff40c3b2c8dbd738a973b0 2024-11-19T12:52:07,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/42cfe581a37e46c9869a20f90c31616c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/C/42cfe581a37e46c9869a20f90c31616c 2024-11-19T12:52:07,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/recovered.edits/518.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4/recovered.edits/518.seqid 2024-11-19T12:52:07,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/86dc49f9832c693612c21abbffe191c4 2024-11-19T12:52:07,142 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:52:07,147 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T12:52:07,159 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:52:07,185 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:52:07,186 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,187 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:52:07,187 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020727187"}]},"ts":"9223372036854775807"} 2024-11-19T12:52:07,192 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:52:07,192 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86dc49f9832c693612c21abbffe191c4, NAME => 'TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:52:07,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:52:07,192 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020727192"}]},"ts":"9223372036854775807"} 2024-11-19T12:52:07,195 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:52:07,196 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-19T12:52:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T12:52:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:52:07,212 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:07,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;aba5a916dfea:35349-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-554143411_22 at /127.0.0.1:60692 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 282) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5871 (was 6310) 2024-11-19T12:52:07,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=5871 2024-11-19T12:52:07,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:52:07,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:52:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:07,242 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:52:07,242 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:07,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-19T12:52:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:07,243 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:52:07,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741953_1129 (size=1061) 2024-11-19T12:52:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:07,654 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:52:07,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741954_1130 (size=53) 2024-11-19T12:52:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:08,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:52:08,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5461aaf01d3071b7aec04e96292d4b32, disabling compactions & flushes 2024-11-19T12:52:08,062 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,062 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. after waiting 0 ms 2024-11-19T12:52:08,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,063 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,063 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5461aaf01d3071b7aec04e96292d4b32: Waiting for close lock at 1732020728062Disabling compacts and flushes for region at 1732020728062Disabling writes for close at 1732020728063 (+1 ms)Writing region close event to WAL at 1732020728063Closed at 1732020728063 2024-11-19T12:52:08,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:52:08,066 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020728065"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020728065"}]},"ts":"1732020728065"} 2024-11-19T12:52:08,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:52:08,072 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:52:08,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020728072"}]},"ts":"1732020728072"} 2024-11-19T12:52:08,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:52:08,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, ASSIGN}] 2024-11-19T12:52:08,076 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, ASSIGN 2024-11-19T12:52:08,077 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:52:08,228 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:08,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, ASSIGN because future has completed 2024-11-19T12:52:08,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:08,396 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,397 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:52:08,397 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,397 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:52:08,398 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,398 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,401 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,403 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:08,404 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName A 2024-11-19T12:52:08,404 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:08,405 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:08,405 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,407 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:08,407 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName B 2024-11-19T12:52:08,407 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:08,408 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:08,408 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,409 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:08,409 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName C 2024-11-19T12:52:08,409 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:08,409 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:08,409 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,410 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,410 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,410 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,412 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,412 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,412 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:52:08,413 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,415 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:52:08,415 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5461aaf01d3071b7aec04e96292d4b32; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73464511, jitterRate=0.09470652043819427}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:52:08,416 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:08,416 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5461aaf01d3071b7aec04e96292d4b32: Running coprocessor pre-open hook at 1732020728398Writing region info on filesystem at 1732020728398Initializing all the Stores at 1732020728400 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020728400Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020728400Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020728400Cleaning up temporary data from old regions at 1732020728412 (+12 ms)Running coprocessor post-open hooks at 1732020728416 (+4 ms)Region opened successfully at 1732020728416 2024-11-19T12:52:08,417 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., pid=34, masterSystemTime=1732020728391 2024-11-19T12:52:08,419 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,419 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:08,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:08,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:08,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T12:52:08,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 in 187 msec 2024-11-19T12:52:08,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T12:52:08,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, ASSIGN in 348 msec 2024-11-19T12:52:08,426 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:52:08,426 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020728426"}]},"ts":"1732020728426"} 2024-11-19T12:52:08,427 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:52:08,428 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:52:08,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-19T12:52:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T12:52:09,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:09,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:09,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:09,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:09,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de59cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:09,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:09,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1fef20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:09,405 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:09,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:09,410 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:52:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:52:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@474a65f 2024-11-19T12:52:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:52:09,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:52:09,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:52:09,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:52:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:09,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741955_1131 (size=1097) 2024-11-19T12:52:09,838 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T12:52:09,838 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T12:52:09,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:52:09,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE}] 2024-11-19T12:52:09,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE 2024-11-19T12:52:09,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:09,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE because future has completed 2024-11-19T12:52:09,851 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:52:09,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:10,004 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,004 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:52:10,005 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5461aaf01d3071b7aec04e96292d4b32, disabling compactions & flushes 2024-11-19T12:52:10,005 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,005 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,005 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. after waiting 0 ms 2024-11-19T12:52:10,005 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,013 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:52:10,014 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,014 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5461aaf01d3071b7aec04e96292d4b32: Waiting for close lock at 1732020730005Running coprocessor pre-close hooks at 1732020730005Disabling compacts and flushes for region at 1732020730005Disabling writes for close at 1732020730005Writing region close event to WAL at 1732020730006 (+1 ms)Running coprocessor post-close hooks at 1732020730014 (+8 ms)Closed at 1732020730014 2024-11-19T12:52:10,014 WARN [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5461aaf01d3071b7aec04e96292d4b32 to self. 2024-11-19T12:52:10,019 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=CLOSED 2024-11-19T12:52:10,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:10,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T12:52:10,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 in 171 msec 2024-11-19T12:52:10,025 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE; state=CLOSED, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=true 2024-11-19T12:52:10,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE because future has completed 2024-11-19T12:52:10,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:10,338 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,338 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:52:10,338 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,338 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:52:10,338 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,338 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,340 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,341 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:10,345 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName A 2024-11-19T12:52:10,347 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:10,348 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:10,348 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,349 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:10,349 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName B 2024-11-19T12:52:10,349 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:10,350 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:10,350 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,350 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:10,351 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5461aaf01d3071b7aec04e96292d4b32 columnFamilyName C 2024-11-19T12:52:10,351 DEBUG [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:10,351 INFO [StoreOpener-5461aaf01d3071b7aec04e96292d4b32-1 {}] regionserver.HStore(327): Store=5461aaf01d3071b7aec04e96292d4b32/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:10,351 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,351 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,352 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,353 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,354 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,354 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,354 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:52:10,356 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,357 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5461aaf01d3071b7aec04e96292d4b32; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58753530, jitterRate=-0.12450417876243591}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:52:10,357 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,358 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5461aaf01d3071b7aec04e96292d4b32: Running coprocessor pre-open hook at 1732020730338Writing region info on filesystem at 1732020730338Initializing all the Stores at 1732020730339 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020730339Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020730339Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020730340 (+1 ms)Cleaning up temporary data from old regions at 1732020730354 (+14 ms)Running coprocessor post-open hooks at 1732020730357 (+3 ms)Region opened successfully at 1732020730358 (+1 ms) 2024-11-19T12:52:10,359 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., pid=39, masterSystemTime=1732020730335 2024-11-19T12:52:10,361 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,361 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,362 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=OPEN, openSeqNum=5, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:10,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-19T12:52:10,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 in 183 msec 2024-11-19T12:52:10,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T12:52:10,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, REOPEN/MOVE in 520 msec 2024-11-19T12:52:10,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T12:52:10,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-19T12:52:10,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 956 msec 2024-11-19T12:52:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T12:52:10,379 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:10,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2253fa83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,383 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2997a437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,385 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6966cdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b7cdb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66149c85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2caabd59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a7efcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fdbb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f6db35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54942632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131ce27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757c90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a56c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dde6639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2620fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:10,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:10,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221cd9ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:10,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9778420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:10,419 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,419 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,419 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,419 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T12:52:10,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:10,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:10,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,431 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,434 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,434 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,435 DEBUG [Thread-664 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,436 DEBUG [Thread-664 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,436 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,437 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:52:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:10,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:10,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c260bf9ebce34532bd25418300494037_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020730446/Put/seqid=0 2024-11-19T12:52:10,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732020730460, max timestamp=1732020730502Num uniques 859; 2024-11-19T12:52:10,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-19T12:52:10,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732020730460, max timestamp=1732020730502Num uniques 859; 2024-11-19T12:52:10,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-19T12:52:10,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1732020730460, max timestamp=1732020730502Num uniques 859; 2024-11-19T12:52:10,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-11-19T12:52:10,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741956_1132 (size=17034) 2024-11-19T12:52:10,529 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:10,539 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c260bf9ebce34532bd25418300494037_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c260bf9ebce34532bd25418300494037_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:10,541 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/2b9aa7d1157a446fb6176f5f8d008704, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020790532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020790530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020790534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/2b9aa7d1157a446fb6176f5f8d008704 is 175, key is test_row_0/A:col10/1732020730446/Put/seqid=0 2024-11-19T12:52:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020790548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020790548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741957_1133 (size=48135) 2024-11-19T12:52:10,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020790652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020790652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020790661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020790672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020790672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:10,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020790862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020790863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020790871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020790892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020790892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:10,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/2b9aa7d1157a446fb6176f5f8d008704 2024-11-19T12:52:11,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/4ed5c764955945a1af1c27f9f61319b7 is 50, key is test_row_0/B:col10/1732020730446/Put/seqid=0 2024-11-19T12:52:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741958_1134 (size=9657) 2024-11-19T12:52:11,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/4ed5c764955945a1af1c27f9f61319b7 2024-11-19T12:52:11,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/3d399d81a84f450892b5d74f635b6256 is 50, key is test_row_0/C:col10/1732020730446/Put/seqid=0 2024-11-19T12:52:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:11,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741959_1135 (size=9657) 2024-11-19T12:52:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020791171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020791171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020791182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:11,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020791201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020791212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:11,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/3d399d81a84f450892b5d74f635b6256 2024-11-19T12:52:11,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/2b9aa7d1157a446fb6176f5f8d008704 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704 2024-11-19T12:52:11,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704, entries=250, sequenceid=14, filesize=47.0 K 2024-11-19T12:52:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/4ed5c764955945a1af1c27f9f61319b7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7 2024-11-19T12:52:11,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7, entries=100, sequenceid=14, filesize=9.4 K 2024-11-19T12:52:11,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/3d399d81a84f450892b5d74f635b6256 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256 2024-11-19T12:52:11,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256, entries=100, sequenceid=14, filesize=9.4 K 2024-11-19T12:52:11,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 5461aaf01d3071b7aec04e96292d4b32 in 1040ms, sequenceid=14, compaction requested=false 2024-11-19T12:52:11,492 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T12:52:11,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:11,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,500 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=154.31 KB heapSize=407.25 KB 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119088f5e3e4a0e499189dfea2f21a83ec8_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020730519/Put/seqid=0 2024-11-19T12:52:11,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741960_1136 (size=12154) 2024-11-19T12:52:11,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,538 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119088f5e3e4a0e499189dfea2f21a83ec8_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119088f5e3e4a0e499189dfea2f21a83ec8_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/4640f00ec58747dcaaf144141df7a149, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:11,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/4640f00ec58747dcaaf144141df7a149 is 175, key is test_row_0/A:col10/1732020730519/Put/seqid=0 2024-11-19T12:52:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741961_1137 (size=30955) 2024-11-19T12:52:11,552 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/4640f00ec58747dcaaf144141df7a149 2024-11-19T12:52:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/72e6b36a0c4940eea258436d25cbd979 is 50, key is test_row_0/B:col10/1732020730519/Put/seqid=0 2024-11-19T12:52:11,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741962_1138 (size=12001) 2024-11-19T12:52:11,618 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/72e6b36a0c4940eea258436d25cbd979 2024-11-19T12:52:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/bf5ff2e945cc454fa5e4826dcc5626f5 is 50, key is test_row_0/C:col10/1732020730519/Put/seqid=0 2024-11-19T12:52:11,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741963_1139 (size=12001) 2024-11-19T12:52:11,657 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/bf5ff2e945cc454fa5e4826dcc5626f5 2024-11-19T12:52:11,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/4640f00ec58747dcaaf144141df7a149 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149 2024-11-19T12:52:11,669 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149, entries=150, sequenceid=46, filesize=30.2 K 2024-11-19T12:52:11,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/72e6b36a0c4940eea258436d25cbd979 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979 2024-11-19T12:52:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,677 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T12:52:11,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/bf5ff2e945cc454fa5e4826dcc5626f5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5 2024-11-19T12:52:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,686 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T12:52:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,687 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~407.20 KB/416976, currentSize=0 B/0 for 5461aaf01d3071b7aec04e96292d4b32 in 187ms, sequenceid=46, compaction requested=false 2024-11-19T12:52:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T12:52:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T12:52:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T12:52:11,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-11-19T12:52:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.2740 sec 2024-11-19T12:52:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:11,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119adb6333a42834d0684b1961a7016602c_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741964_1140 (size=21918) 2024-11-19T12:52:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119adb6333a42834d0684b1961a7016602c_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adb6333a42834d0684b1961a7016602c_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/7834a0dd78134addb0c03e73742a5088, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:11,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/7834a0dd78134addb0c03e73742a5088 is 175, key is test_row_0/A:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741965_1141 (size=65323) 2024-11-19T12:52:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,816 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/7834a0dd78134addb0c03e73742a5088 2024-11-19T12:52:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732020731754, max timestamp=1732020731815Num uniques 859; 2024-11-19T12:52:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T12:52:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732020731754, max timestamp=1732020731815Num uniques 859; 2024-11-19T12:52:11,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T12:52:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732020731754, max timestamp=1732020731815Num uniques 859; 2024-11-19T12:52:11,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-19T12:52:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020791820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020791822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020791823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/df11648bf05f4d3b9f5d6f0f54f496b9 is 50, key is test_row_0/B:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020791829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020791831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741966_1142 (size=21365) 2024-11-19T12:52:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020791942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,097 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:52:12,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020792167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/df11648bf05f4d3b9f5d6f0f54f496b9 2024-11-19T12:52:12,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/f84cbc6017b74466ac6db274820d56f9 is 50, key is test_row_0/C:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:12,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741967_1143 (size=12001) 2024-11-19T12:52:12,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/f84cbc6017b74466ac6db274820d56f9 2024-11-19T12:52:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/7834a0dd78134addb0c03e73742a5088 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088 2024-11-19T12:52:12,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088, entries=350, sequenceid=56, filesize=63.8 K 2024-11-19T12:52:12,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/df11648bf05f4d3b9f5d6f0f54f496b9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9 2024-11-19T12:52:12,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9, entries=350, sequenceid=56, filesize=20.9 K 2024-11-19T12:52:12,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/f84cbc6017b74466ac6db274820d56f9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9 2024-11-19T12:52:12,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9, entries=150, sequenceid=56, filesize=11.7 K 2024-11-19T12:52:12,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 5461aaf01d3071b7aec04e96292d4b32 in 596ms, sequenceid=56, compaction requested=true 2024-11-19T12:52:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:12,349 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:12,350 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:12,351 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:12,351 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:12,351 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,351 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=141.0 K 2024-11-19T12:52:12,351 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,352 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088] 2024-11-19T12:52:12,352 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:12,352 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:12,352 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,352 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=42.0 K 2024-11-19T12:52:12,353 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9aa7d1157a446fb6176f5f8d008704, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020730438 2024-11-19T12:52:12,353 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed5c764955945a1af1c27f9f61319b7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020730446 2024-11-19T12:52:12,353 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4640f00ec58747dcaaf144141df7a149, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020730519 2024-11-19T12:52:12,353 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e6b36a0c4940eea258436d25cbd979, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020730519 2024-11-19T12:52:12,354 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7834a0dd78134addb0c03e73742a5088, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731736 2024-11-19T12:52:12,354 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting df11648bf05f4d3b9f5d6f0f54f496b9, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731736 2024-11-19T12:52:12,368 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:12,369 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/366790f02b6543809aecb70c3eac9b67 is 50, key is test_row_0/B:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:12,371 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:12,383 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197a8187aa188342b7b1228744d898a9c8_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:12,386 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197a8187aa188342b7b1228744d898a9c8_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:12,386 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a8187aa188342b7b1228744d898a9c8_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:12,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741968_1144 (size=12104) 2024-11-19T12:52:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741969_1145 (size=4469) 2024-11-19T12:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:12,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=167.72 KB heapSize=418.97 KB 2024-11-19T12:52:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:12,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:12,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:12,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020792493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020792494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020792495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190b01727e61944e249d37cd24f62102ab_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020732486/Put/seqid=0 2024-11-19T12:52:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020792496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020792497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741970_1146 (size=12154) 2024-11-19T12:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T12:52:12,571 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:12,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T12:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:12,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:12,582 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:12,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:12,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020792612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020792612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020792612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020792612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020792612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:12,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,797 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/366790f02b6543809aecb70c3eac9b67 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/366790f02b6543809aecb70c3eac9b67 2024-11-19T12:52:12,805 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#126 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:12,806 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/11a223ba845541438c1fc9fb9bf0ed57 is 175, key is test_row_0/A:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:12,808 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into 366790f02b6543809aecb70c3eac9b67(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:12,809 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:12,809 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020732350; duration=0sec 2024-11-19T12:52:12,809 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:12,809 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:12,809 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:12,810 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:12,810 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:12,811 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,811 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=32.9 K 2024-11-19T12:52:12,811 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d399d81a84f450892b5d74f635b6256, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020730446 2024-11-19T12:52:12,812 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5ff2e945cc454fa5e4826dcc5626f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020730519 2024-11-19T12:52:12,812 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f84cbc6017b74466ac6db274820d56f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731737 2024-11-19T12:52:12,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741971_1147 (size=31058) 2024-11-19T12:52:12,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020792822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020792822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020792822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020792822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020792822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:12,835 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/11a223ba845541438c1fc9fb9bf0ed57 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57 2024-11-19T12:52:12,835 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:12,836 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/c904e1948f544b0dad71917be5e72dba is 50, key is test_row_0/C:col10/1732020731737/Put/seqid=0 2024-11-19T12:52:12,845 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into 11a223ba845541438c1fc9fb9bf0ed57(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:12,845 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:12,845 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020732349; duration=0sec 2024-11-19T12:52:12,845 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:12,845 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:12,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741972_1148 (size=12104) 2024-11-19T12:52:12,867 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/c904e1948f544b0dad71917be5e72dba as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/c904e1948f544b0dad71917be5e72dba 2024-11-19T12:52:12,874 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into c904e1948f544b0dad71917be5e72dba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:12,874 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:12,874 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020732350; duration=0sec 2024-11-19T12:52:12,875 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:12,875 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:12,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:12,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:12,925 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:12,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190b01727e61944e249d37cd24f62102ab_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b01727e61944e249d37cd24f62102ab_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:12,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/56b9c7b6491b48d48522745201382bf0, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:12,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/56b9c7b6491b48d48522745201382bf0 is 175, key is test_row_0/A:col10/1732020732486/Put/seqid=0 2024-11-19T12:52:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741973_1149 (size=30955) 2024-11-19T12:52:12,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/56b9c7b6491b48d48522745201382bf0 2024-11-19T12:52:12,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/1a3b570781ea436daacb6b6c51f71005 is 50, key is test_row_0/B:col10/1732020732486/Put/seqid=0 2024-11-19T12:52:12,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741974_1150 (size=12001) 2024-11-19T12:52:12,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/1a3b570781ea436daacb6b6c51f71005 2024-11-19T12:52:12,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/87dde872f0d34fa1959cac9cbf118fb4 is 50, key is test_row_0/C:col10/1732020732486/Put/seqid=0 2024-11-19T12:52:13,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741975_1151 (size=12001) 2024-11-19T12:52:13,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/87dde872f0d34fa1959cac9cbf118fb4 2024-11-19T12:52:13,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/56b9c7b6491b48d48522745201382bf0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0 2024-11-19T12:52:13,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0, entries=150, sequenceid=85, filesize=30.2 K 2024-11-19T12:52:13,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/1a3b570781ea436daacb6b6c51f71005 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005 2024-11-19T12:52:13,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T12:52:13,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/87dde872f0d34fa1959cac9cbf118fb4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4 2024-11-19T12:52:13,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T12:52:13,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~438.84 KB/449376, currentSize=33.54 KB/34350 for 5461aaf01d3071b7aec04e96292d4b32 in 569ms, sequenceid=85, compaction requested=false 2024-11-19T12:52:13,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:13,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:52:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:13,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:13,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:13,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111922ab113a741245a3b3e92d9fbba50947_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:13,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732020733154, max timestamp=1732020733167Num uniques 859; 2024-11-19T12:52:13,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732020733154, max timestamp=1732020733167Num uniques 859; 2024-11-19T12:52:13,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T12:52:13,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T12:52:13,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732020733154, max timestamp=1732020733167Num uniques 859; 2024-11-19T12:52:13,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T12:52:13,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741976_1152 (size=12154) 2024-11-19T12:52:13,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020793189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020793189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020793190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020793213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020793213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020793321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020793322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020793322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020793322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020793322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,576 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:13,580 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111922ab113a741245a3b3e92d9fbba50947_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922ab113a741245a3b3e92d9fbba50947_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:13,582 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d6eb5e8e778f4299afea19cedcb86a49, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:13,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d6eb5e8e778f4299afea19cedcb86a49 is 175, key is test_row_0/A:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:13,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741977_1153 (size=30955) 2024-11-19T12:52:13,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:13,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020793842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020793842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020793842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020793842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020793842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:13,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:13,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:13,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d6eb5e8e778f4299afea19cedcb86a49 2024-11-19T12:52:14,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b055ca2372f64d289e0c1b4e08a69310 is 50, key is test_row_0/B:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:14,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741978_1154 (size=12001) 2024-11-19T12:52:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020794351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020794351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020794352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020794352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020794352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:14,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b055ca2372f64d289e0c1b4e08a69310 2024-11-19T12:52:14,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2e439cd3467e4daeb030420d291f0cf4 is 50, key is test_row_0/C:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:14,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:14,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741979_1155 (size=12001) 2024-11-19T12:52:14,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2e439cd3467e4daeb030420d291f0cf4 2024-11-19T12:52:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d6eb5e8e778f4299afea19cedcb86a49 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49 2024-11-19T12:52:14,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49, entries=150, sequenceid=99, filesize=30.2 K 2024-11-19T12:52:14,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b055ca2372f64d289e0c1b4e08a69310 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310 2024-11-19T12:52:14,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310, entries=150, sequenceid=99, filesize=11.7 K 2024-11-19T12:52:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2e439cd3467e4daeb030420d291f0cf4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4 2024-11-19T12:52:14,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4, entries=150, sequenceid=99, filesize=11.7 K 2024-11-19T12:52:14,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5461aaf01d3071b7aec04e96292d4b32 in 1329ms, sequenceid=99, compaction requested=true 2024-11-19T12:52:14,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:14,465 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:14,465 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:14,466 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:14,466 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:14,467 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,467 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=90.8 K 2024-11-19T12:52:14,467 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,467 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49] 2024-11-19T12:52:14,467 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:14,467 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:14,467 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,467 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/366790f02b6543809aecb70c3eac9b67, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.3 K 2024-11-19T12:52:14,467 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a223ba845541438c1fc9fb9bf0ed57, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731737 2024-11-19T12:52:14,468 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 366790f02b6543809aecb70c3eac9b67, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731737 2024-11-19T12:52:14,468 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b9c7b6491b48d48522745201382bf0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020731797 2024-11-19T12:52:14,469 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a3b570781ea436daacb6b6c51f71005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020731797 2024-11-19T12:52:14,469 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6eb5e8e778f4299afea19cedcb86a49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:14,470 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b055ca2372f64d289e0c1b4e08a69310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:14,481 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:14,483 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111988946f69753b496abe6a5877b69a4d80_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:14,484 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:14,485 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/419efda2eb024e6fb4f7b20e86b5b317 is 50, key is test_row_0/B:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:14,485 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111988946f69753b496abe6a5877b69a4d80_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:14,486 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111988946f69753b496abe6a5877b69a4d80_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:14,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741981_1157 (size=4469) 2024-11-19T12:52:14,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741980_1156 (size=12207) 2024-11-19T12:52:14,500 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#134 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:14,501 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/923c58e494b54ad9820b7f8d9ef8b037 is 175, key is test_row_0/A:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:14,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741982_1158 (size=31161) 2024-11-19T12:52:14,520 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/923c58e494b54ad9820b7f8d9ef8b037 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037 2024-11-19T12:52:14,527 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into 923c58e494b54ad9820b7f8d9ef8b037(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:14,528 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:14,528 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020734464; duration=0sec 2024-11-19T12:52:14,528 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:14,528 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:14,528 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:14,530 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:14,530 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:14,530 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,530 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/c904e1948f544b0dad71917be5e72dba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.3 K 2024-11-19T12:52:14,530 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c904e1948f544b0dad71917be5e72dba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732020731737 2024-11-19T12:52:14,531 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87dde872f0d34fa1959cac9cbf118fb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732020731797 2024-11-19T12:52:14,532 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e439cd3467e4daeb030420d291f0cf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:14,544 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:14,544 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/dda9ac4d8fc44b48907ecef9f08208b6 is 50, key is test_row_0/C:col10/1732020733136/Put/seqid=0 2024-11-19T12:52:14,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741983_1159 (size=12207) 2024-11-19T12:52:14,567 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/dda9ac4d8fc44b48907ecef9f08208b6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/dda9ac4d8fc44b48907ecef9f08208b6 2024-11-19T12:52:14,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T12:52:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:14,572 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-19T12:52:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:14,579 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into dda9ac4d8fc44b48907ecef9f08208b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:14,579 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:14,579 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020734465; duration=0sec 2024-11-19T12:52:14,579 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:14,579 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111913124cc90e754680b82f009dc9dc7767_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020733193/Put/seqid=0 2024-11-19T12:52:14,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741984_1160 (size=12304) 2024-11-19T12:52:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:14,906 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/419efda2eb024e6fb4f7b20e86b5b317 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/419efda2eb024e6fb4f7b20e86b5b317 2024-11-19T12:52:14,913 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into 419efda2eb024e6fb4f7b20e86b5b317(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:14,914 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:14,914 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020734465; duration=0sec 2024-11-19T12:52:14,914 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:14,914 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:14,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:15,000 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111913124cc90e754680b82f009dc9dc7767_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111913124cc90e754680b82f009dc9dc7767_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/96eaa8bc38ee4719b780e20d6fa95a03, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/96eaa8bc38ee4719b780e20d6fa95a03 is 175, key is test_row_0/A:col10/1732020733193/Put/seqid=0 2024-11-19T12:52:15,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741985_1161 (size=31105) 2024-11-19T12:52:15,020 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/96eaa8bc38ee4719b780e20d6fa95a03 2024-11-19T12:52:15,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/207ba886981346ae8935cb3581c9906e is 50, key is test_row_0/B:col10/1732020733193/Put/seqid=0 2024-11-19T12:52:15,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741986_1162 (size=12151) 2024-11-19T12:52:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020795369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020795371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020795411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020795412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020795412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,441 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/207ba886981346ae8935cb3581c9906e 2024-11-19T12:52:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/76b8cc72ecb147c19e80e2ce9405f3b2 is 50, key is test_row_0/C:col10/1732020733193/Put/seqid=0 2024-11-19T12:52:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741987_1163 (size=12151) 2024-11-19T12:52:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020795522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020795522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020795522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020795522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020795522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020795731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020795731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020795732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020795732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020795732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:15,863 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/76b8cc72ecb147c19e80e2ce9405f3b2 2024-11-19T12:52:15,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/96eaa8bc38ee4719b780e20d6fa95a03 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03 2024-11-19T12:52:15,878 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03, entries=150, sequenceid=133, filesize=30.4 K 2024-11-19T12:52:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/207ba886981346ae8935cb3581c9906e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e 2024-11-19T12:52:15,886 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e, entries=150, sequenceid=133, filesize=11.9 K 2024-11-19T12:52:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/76b8cc72ecb147c19e80e2ce9405f3b2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2 2024-11-19T12:52:15,904 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2, entries=150, sequenceid=133, filesize=11.9 K 2024-11-19T12:52:15,905 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=60.38 KB/61830 for 5461aaf01d3071b7aec04e96292d4b32 in 1333ms, sequenceid=133, compaction requested=false 2024-11-19T12:52:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T12:52:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T12:52:15,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T12:52:15,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3250 sec 2024-11-19T12:52:15,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.3390 sec 2024-11-19T12:52:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:16,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:52:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:16,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:16,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:16,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:16,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0d847dab2f24074af990c99592b4843_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741988_1164 (size=14794) 2024-11-19T12:52:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020796086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020796093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020796094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020796096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020796097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020796201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020796212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020796212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020796212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020796212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020796412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020796421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020796421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020796421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020796421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:16,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0d847dab2f24074af990c99592b4843_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0d847dab2f24074af990c99592b4843_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:16,462 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/c5ad0e1c36b343aea6732b22379a6ab9, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:16,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/c5ad0e1c36b343aea6732b22379a6ab9 is 175, key is test_row_0/A:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:16,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741989_1165 (size=39749) 2024-11-19T12:52:16,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020796721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020796731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020796731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020796731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020796732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T12:52:16,751 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:16,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T12:52:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:16,753 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:16,754 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:16,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:16,869 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/c5ad0e1c36b343aea6732b22379a6ab9 2024-11-19T12:52:16,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/f04c4fc29273496a91a4f753e4453cb0 is 50, key is test_row_0/B:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741990_1166 (size=12151) 2024-11-19T12:52:16,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/f04c4fc29273496a91a4f753e4453cb0 2024-11-19T12:52:16,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/7331f460c8034df88c264a9cd4db1675 is 50, key is test_row_0/C:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:16,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741991_1167 (size=12151) 2024-11-19T12:52:16,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T12:52:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:16,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T12:52:17,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:17,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:17,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T12:52:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:17,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020797232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020797241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:17,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/7331f460c8034df88c264a9cd4db1675 2024-11-19T12:52:17,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/c5ad0e1c36b343aea6732b22379a6ab9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9 2024-11-19T12:52:17,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9, entries=200, sequenceid=148, filesize=38.8 K 2024-11-19T12:52:17,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/f04c4fc29273496a91a4f753e4453cb0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0 2024-11-19T12:52:17,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0, entries=150, sequenceid=148, filesize=11.9 K 2024-11-19T12:52:17,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/7331f460c8034df88c264a9cd4db1675 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675 2024-11-19T12:52:17,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675, entries=150, sequenceid=148, filesize=11.9 K 2024-11-19T12:52:17,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5461aaf01d3071b7aec04e96292d4b32 in 1271ms, sequenceid=148, compaction requested=true 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:17,314 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:17,314 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:17,315 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:17,315 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:17,316 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:17,316 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,316 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/419efda2eb024e6fb4f7b20e86b5b317, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.7 K 2024-11-19T12:52:17,316 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:17,316 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,316 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=99.6 K 2024-11-19T12:52:17,316 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,316 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9] 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923c58e494b54ad9820b7f8d9ef8b037, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 419efda2eb024e6fb4f7b20e86b5b317, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 207ba886981346ae8935cb3581c9906e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732020733188 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96eaa8bc38ee4719b780e20d6fa95a03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732020733188 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f04c4fc29273496a91a4f753e4453cb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735375 2024-11-19T12:52:17,317 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ad0e1c36b343aea6732b22379a6ab9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735368 2024-11-19T12:52:17,323 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:17,324 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:17,325 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/dfec101f76c6496491611074c8455a94 is 50, key is test_row_0/B:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:17,326 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198d4ff1c74b60488db68cc2753d48378f_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:17,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198d4ff1c74b60488db68cc2753d48378f_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:17,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d4ff1c74b60488db68cc2753d48378f_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:17,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741993_1169 (size=12459) 2024-11-19T12:52:17,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741992_1168 (size=4469) 2024-11-19T12:52:17,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,365 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119671cb05a4140451a81a5392044e0c241_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020736063/Put/seqid=0 2024-11-19T12:52:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741994_1170 (size=12304) 2024-11-19T12:52:17,737 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/dfec101f76c6496491611074c8455a94 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/dfec101f76c6496491611074c8455a94 2024-11-19T12:52:17,738 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:17,739 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d401ba41640b4c108655a0a0a57289a0 is 175, key is test_row_0/A:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:17,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741995_1171 (size=31413) 2024-11-19T12:52:17,745 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into dfec101f76c6496491611074c8455a94(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:17,745 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:17,745 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020737314; duration=0sec 2024-11-19T12:52:17,745 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:17,745 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:17,745 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:17,746 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:17,746 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:17,746 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:17,747 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/dda9ac4d8fc44b48907ecef9f08208b6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.7 K 2024-11-19T12:52:17,747 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting dda9ac4d8fc44b48907ecef9f08208b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732020732490 2024-11-19T12:52:17,747 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 76b8cc72ecb147c19e80e2ce9405f3b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732020733188 2024-11-19T12:52:17,748 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7331f460c8034df88c264a9cd4db1675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735375 2024-11-19T12:52:17,749 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/d401ba41640b4c108655a0a0a57289a0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0 2024-11-19T12:52:17,755 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into d401ba41640b4c108655a0a0a57289a0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:17,755 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:17,755 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020737314; duration=0sec 2024-11-19T12:52:17,755 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#146 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:17,755 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:17,755 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:17,756 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/23d6ddbd072c4187ba242c4a4253f1c5 is 50, key is test_row_0/C:col10/1732020735375/Put/seqid=0 2024-11-19T12:52:17,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741996_1172 (size=12459) 2024-11-19T12:52:17,770 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/23d6ddbd072c4187ba242c4a4253f1c5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/23d6ddbd072c4187ba242c4a4253f1c5 2024-11-19T12:52:17,775 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into 23d6ddbd072c4187ba242c4a4253f1c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:17,775 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:17,775 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020737314; duration=0sec 2024-11-19T12:52:17,775 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:17,775 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:17,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:17,797 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119671cb05a4140451a81a5392044e0c241_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119671cb05a4140451a81a5392044e0c241_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/623da7f2149443e1812c0234a793f9a1, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/623da7f2149443e1812c0234a793f9a1 is 175, key is test_row_0/A:col10/1732020736063/Put/seqid=0 2024-11-19T12:52:17,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741997_1173 (size=31105) 2024-11-19T12:52:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:18,203 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/623da7f2149443e1812c0234a793f9a1 2024-11-19T12:52:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ae17fb105dd24ca0b67bb6ad724b7824 is 50, key is test_row_0/B:col10/1732020736063/Put/seqid=0 2024-11-19T12:52:18,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741998_1174 (size=12151) 2024-11-19T12:52:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020798258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020798259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020798261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020798261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020798262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,266 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 5461aaf01d3071b7aec04e96292d4b32 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020798371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020798371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020798372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020798372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020798581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020798581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020798582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020798582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,620 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ae17fb105dd24ca0b67bb6ad724b7824 2024-11-19T12:52:18,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/aa656ea4c24042bc9f9dd3df0f8392d4 is 50, key is test_row_0/C:col10/1732020736063/Put/seqid=0 2024-11-19T12:52:18,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741999_1175 (size=12151) 2024-11-19T12:52:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020798891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:19,031 INFO [master/aba5a916dfea:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T12:52:19,031 INFO [master/aba5a916dfea:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T12:52:19,033 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/aa656ea4c24042bc9f9dd3df0f8392d4 2024-11-19T12:52:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/623da7f2149443e1812c0234a793f9a1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1 2024-11-19T12:52:19,042 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1, entries=150, sequenceid=167, filesize=30.4 K 2024-11-19T12:52:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ae17fb105dd24ca0b67bb6ad724b7824 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824 2024-11-19T12:52:19,048 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T12:52:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/aa656ea4c24042bc9f9dd3df0f8392d4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4 2024-11-19T12:52:19,053 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T12:52:19,055 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 5461aaf01d3071b7aec04e96292d4b32 in 1690ms, sequenceid=167, compaction requested=false 2024-11-19T12:52:19,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:19,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:19,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T12:52:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T12:52:19,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T12:52:19,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3020 sec 2024-11-19T12:52:19,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3080 sec 2024-11-19T12:52:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:19,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:19,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:19,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923568365b7f148d598a297186658bd1d_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:19,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742000_1176 (size=12304) 2024-11-19T12:52:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020799421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020799422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020799422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020799424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020799531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020799531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020799531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020799541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020799741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020799741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020799741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020799752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:19,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:19,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923568365b7f148d598a297186658bd1d_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923568365b7f148d598a297186658bd1d_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:19,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b348a2943c0c42099112bc3b3da1a34c, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:19,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b348a2943c0c42099112bc3b3da1a34c is 175, key is test_row_0/A:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742001_1177 (size=31105) 2024-11-19T12:52:20,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020800051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020800051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020800051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020800061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,230 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b348a2943c0c42099112bc3b3da1a34c 2024-11-19T12:52:20,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/17aefc8ba6884255a2b749c114c32804 is 50, key is test_row_0/B:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:20,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742002_1178 (size=12151) 2024-11-19T12:52:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020800281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,283 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 5461aaf01d3071b7aec04e96292d4b32 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020800561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020800561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020800561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020800571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:20,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/17aefc8ba6884255a2b749c114c32804 2024-11-19T12:52:20,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2242af58acc4482493ec89b2cce3af6b is 50, key is test_row_0/C:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:20,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742003_1179 (size=12151) 2024-11-19T12:52:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T12:52:20,912 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:20,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T12:52:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:20,913 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:20,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:20,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:21,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T12:52:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:21,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2242af58acc4482493ec89b2cce3af6b 2024-11-19T12:52:21,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b348a2943c0c42099112bc3b3da1a34c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c 2024-11-19T12:52:21,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c, entries=150, sequenceid=186, filesize=30.4 K 2024-11-19T12:52:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/17aefc8ba6884255a2b749c114c32804 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804 2024-11-19T12:52:21,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T12:52:21,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/2242af58acc4482493ec89b2cce3af6b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b 2024-11-19T12:52:21,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T12:52:21,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5461aaf01d3071b7aec04e96292d4b32 in 1698ms, sequenceid=186, compaction requested=true 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:21,103 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:21,103 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:21,106 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:21,106 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:21,106 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:21,106 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:21,106 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,107 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,107 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=91.4 K 2024-11-19T12:52:21,107 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/dfec101f76c6496491611074c8455a94, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.9 K 2024-11-19T12:52:21,107 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,107 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c] 2024-11-19T12:52:21,107 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d401ba41640b4c108655a0a0a57289a0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735375 2024-11-19T12:52:21,107 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting dfec101f76c6496491611074c8455a94, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735375 2024-11-19T12:52:21,108 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 623da7f2149443e1812c0234a793f9a1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732020736063 2024-11-19T12:52:21,108 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ae17fb105dd24ca0b67bb6ad724b7824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732020736063 2024-11-19T12:52:21,108 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b348a2943c0c42099112bc3b3da1a34c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:21,108 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 17aefc8ba6884255a2b749c114c32804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:21,113 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:21,115 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111981d45a73995242d6a39a2d940c65584b_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:21,115 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:21,116 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/1103bee56e344ab9afdf7cd91e5feeca is 50, key is test_row_0/B:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:21,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111981d45a73995242d6a39a2d940c65584b_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:21,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111981d45a73995242d6a39a2d940c65584b_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742005_1181 (size=4469) 2024-11-19T12:52:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742004_1180 (size=12561) 2024-11-19T12:52:21,133 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/1103bee56e344ab9afdf7cd91e5feeca as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1103bee56e344ab9afdf7cd91e5feeca 2024-11-19T12:52:21,138 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into 1103bee56e344ab9afdf7cd91e5feeca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:21,138 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:21,138 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020741103; duration=0sec 2024-11-19T12:52:21,138 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:21,138 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:21,139 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:21,140 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:21,140 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:21,140 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,140 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/23d6ddbd072c4187ba242c4a4253f1c5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=35.9 K 2024-11-19T12:52:21,140 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d6ddbd072c4187ba242c4a4253f1c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732020735375 2024-11-19T12:52:21,140 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting aa656ea4c24042bc9f9dd3df0f8392d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732020736063 2024-11-19T12:52:21,141 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2242af58acc4482493ec89b2cce3af6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:21,148 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:21,148 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/3bbc171c81cf430c86a643c710851965 is 50, key is test_row_0/C:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:21,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742006_1182 (size=12561) 2024-11-19T12:52:21,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T12:52:21,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:21,219 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912a10b72e3084a6f897d5295827f65e0_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020739422/Put/seqid=0 2024-11-19T12:52:21,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742007_1183 (size=12304) 2024-11-19T12:52:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:21,524 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:21,524 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/1d24ec6e3a5241d5a33761d47b7ec492 is 175, key is test_row_0/A:col10/1732020739403/Put/seqid=0 2024-11-19T12:52:21,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742008_1184 (size=31515) 2024-11-19T12:52:21,533 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/1d24ec6e3a5241d5a33761d47b7ec492 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492 2024-11-19T12:52:21,538 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into 1d24ec6e3a5241d5a33761d47b7ec492(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:21,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:21,538 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020741103; duration=0sec 2024-11-19T12:52:21,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:21,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:21,562 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/3bbc171c81cf430c86a643c710851965 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3bbc171c81cf430c86a643c710851965 2024-11-19T12:52:21,569 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into 3bbc171c81cf430c86a643c710851965(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:21,569 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:21,569 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020741103; duration=0sec 2024-11-19T12:52:21,569 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:21,569 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:21,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:21,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020801621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020801622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020801623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020801624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:21,633 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912a10b72e3084a6f897d5295827f65e0_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912a10b72e3084a6f897d5295827f65e0_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/260a0a3dbee74ff1861cab6d251e0e9a, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/260a0a3dbee74ff1861cab6d251e0e9a is 175, key is test_row_0/A:col10/1732020739422/Put/seqid=0 2024-11-19T12:52:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742009_1185 (size=31105) 2024-11-19T12:52:21,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020801731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020801731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020801731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020801731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020801941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020801941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020801941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020801941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,059 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/260a0a3dbee74ff1861cab6d251e0e9a 2024-11-19T12:52:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/c5e3cc967bca4f4ebb1bb142d4a83c15 is 50, key is test_row_0/B:col10/1732020739422/Put/seqid=0 2024-11-19T12:52:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742010_1186 (size=12151) 2024-11-19T12:52:22,082 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/c5e3cc967bca4f4ebb1bb142d4a83c15 2024-11-19T12:52:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/00cb988a0565475db8e557b4a716d2f3 is 50, key is test_row_0/C:col10/1732020739422/Put/seqid=0 2024-11-19T12:52:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742011_1187 (size=12151) 2024-11-19T12:52:22,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020802251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020802251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020802251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020802251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,500 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/00cb988a0565475db8e557b4a716d2f3 2024-11-19T12:52:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/260a0a3dbee74ff1861cab6d251e0e9a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a 2024-11-19T12:52:22,509 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a, entries=150, sequenceid=203, filesize=30.4 K 2024-11-19T12:52:22,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/c5e3cc967bca4f4ebb1bb142d4a83c15 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15 2024-11-19T12:52:22,513 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T12:52:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/00cb988a0565475db8e557b4a716d2f3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3 2024-11-19T12:52:22,518 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T12:52:22,519 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5461aaf01d3071b7aec04e96292d4b32 in 1300ms, sequenceid=203, compaction requested=false 2024-11-19T12:52:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T12:52:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T12:52:22,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T12:52:22,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6060 sec 2024-11-19T12:52:22,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6100 sec 2024-11-19T12:52:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:22,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:22,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:22,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192917be66880f48608e3244d4d433996c_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:22,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742012_1188 (size=14794) 2024-11-19T12:52:22,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020802778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020802779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020802780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020802781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020802891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020802892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020802892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:22,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020802892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T12:52:23,071 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:23,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T12:52:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:23,073 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:23,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:23,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020803101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020803102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:23,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192917be66880f48608e3244d4d433996c_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192917be66880f48608e3244d4d433996c_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:23,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f21177f4a9545efb52ca1554765a90c, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:23,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f21177f4a9545efb52ca1554765a90c is 175, key is test_row_0/A:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742013_1189 (size=39749) 2024-11-19T12:52:23,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:23,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020803411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020803412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020803412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020803412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f21177f4a9545efb52ca1554765a90c 2024-11-19T12:52:23,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/e8c2bd1171de463b8878ab647e9f2294 is 50, key is test_row_0/B:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:23,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742014_1190 (size=12151) 2024-11-19T12:52:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:23,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020803922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020803922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020803922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020803922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:23,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:23,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/e8c2bd1171de463b8878ab647e9f2294 2024-11-19T12:52:24,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/4a4251efe25f445cb831043c96ab2d89 is 50, key is test_row_0/C:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742015_1191 (size=12151) 2024-11-19T12:52:24,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:24,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:24,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34226 deadline: 1732020804301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:24,307 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 5461aaf01d3071b7aec04e96292d4b32 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:24,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/4a4251efe25f445cb831043c96ab2d89 2024-11-19T12:52:24,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f21177f4a9545efb52ca1554765a90c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c 2024-11-19T12:52:24,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c, entries=200, sequenceid=222, filesize=38.8 K 2024-11-19T12:52:24,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/e8c2bd1171de463b8878ab647e9f2294 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294 2024-11-19T12:52:24,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T12:52:24,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/4a4251efe25f445cb831043c96ab2d89 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89 2024-11-19T12:52:24,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T12:52:24,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5461aaf01d3071b7aec04e96292d4b32 in 1683ms, sequenceid=222, compaction requested=true 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:24,445 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:24,445 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:24,446 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,446 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,446 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1103bee56e344ab9afdf7cd91e5feeca, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=36.0 K 2024-11-19T12:52:24,446 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=100.0 K 2024-11-19T12:52:24,446 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c] 2024-11-19T12:52:24,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1103bee56e344ab9afdf7cd91e5feeca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:24,447 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d24ec6e3a5241d5a33761d47b7ec492, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:24,447 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e3cc967bca4f4ebb1bb142d4a83c15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732020739419 2024-11-19T12:52:24,447 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260a0a3dbee74ff1861cab6d251e0e9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732020739419 2024-11-19T12:52:24,447 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c2bd1171de463b8878ab647e9f2294, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:24,447 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f21177f4a9545efb52ca1554765a90c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:24,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T12:52:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,451 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:24,455 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:24,458 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e0e4c05fcebc4a24af4c739ddd0f63c5_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:24,460 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e0e4c05fcebc4a24af4c739ddd0f63c5_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:24,460 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e0e4c05fcebc4a24af4c739ddd0f63c5_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9c928cabbd54d3a805808037d6b394e_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020742769/Put/seqid=0 2024-11-19T12:52:24,469 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:24,470 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/f7000bfb17704148b09d772ac3fa289d is 50, key is test_row_0/B:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742017_1193 (size=4469) 2024-11-19T12:52:24,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742016_1192 (size=12304) 2024-11-19T12:52:24,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742018_1194 (size=12663) 2024-11-19T12:52:24,882 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#161 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:24,883 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/e1f834e1c81748c68ec34e0e924aede6 is 175, key is test_row_0/A:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:24,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742019_1195 (size=31617) 2024-11-19T12:52:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:24,894 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9c928cabbd54d3a805808037d6b394e_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9c928cabbd54d3a805808037d6b394e_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/705d6c35a9794ebdaf2156212d764448, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/705d6c35a9794ebdaf2156212d764448 is 175, key is test_row_0/A:col10/1732020742769/Put/seqid=0 2024-11-19T12:52:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742020_1196 (size=31105) 2024-11-19T12:52:24,906 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/705d6c35a9794ebdaf2156212d764448 2024-11-19T12:52:24,913 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/f7000bfb17704148b09d772ac3fa289d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f7000bfb17704148b09d772ac3fa289d 2024-11-19T12:52:24,920 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into f7000bfb17704148b09d772ac3fa289d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:24,920 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:24,920 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020744445; duration=0sec 2024-11-19T12:52:24,920 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:24,921 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:24,921 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:24,922 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:24,922 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:24,922 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:24,922 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3bbc171c81cf430c86a643c710851965, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=36.0 K 2024-11-19T12:52:24,925 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bbc171c81cf430c86a643c710851965, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732020738260 2024-11-19T12:52:24,926 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 00cb988a0565475db8e557b4a716d2f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732020739419 2024-11-19T12:52:24,926 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a4251efe25f445cb831043c96ab2d89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:24,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/05a5b97281f04a80b576e23c811afffc is 50, key is test_row_0/B:col10/1732020742769/Put/seqid=0 2024-11-19T12:52:24,939 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:24,940 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/fb6246d1d9df4026987a69a8f8f1189d is 50, key is test_row_0/C:col10/1732020741587/Put/seqid=0 2024-11-19T12:52:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:24,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:24,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742021_1197 (size=12151) 2024-11-19T12:52:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020804964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020804964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020804966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020804966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:24,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742022_1198 (size=12663) 2024-11-19T12:52:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020805072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020805072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020805081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020805081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:25,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020805281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020805281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,293 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/e1f834e1c81748c68ec34e0e924aede6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6 2024-11-19T12:52:25,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020805291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020805291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,297 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into e1f834e1c81748c68ec34e0e924aede6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:25,297 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:25,298 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020744445; duration=0sec 2024-11-19T12:52:25,298 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:25,298 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:25,360 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/05a5b97281f04a80b576e23c811afffc 2024-11-19T12:52:25,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6674485a9d74ce7812968f1f658e17b is 50, key is test_row_0/C:col10/1732020742769/Put/seqid=0 2024-11-19T12:52:25,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742023_1199 (size=12151) 2024-11-19T12:52:25,381 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/fb6246d1d9df4026987a69a8f8f1189d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/fb6246d1d9df4026987a69a8f8f1189d 2024-11-19T12:52:25,386 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into fb6246d1d9df4026987a69a8f8f1189d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:25,386 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:25,386 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020744445; duration=0sec 2024-11-19T12:52:25,386 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:25,386 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020805591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020805591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020805602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020805602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6674485a9d74ce7812968f1f658e17b 2024-11-19T12:52:25,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/705d6c35a9794ebdaf2156212d764448 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448 2024-11-19T12:52:25,823 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448, entries=150, sequenceid=237, filesize=30.4 K 2024-11-19T12:52:25,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/05a5b97281f04a80b576e23c811afffc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc 2024-11-19T12:52:25,828 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T12:52:25,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6674485a9d74ce7812968f1f658e17b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b 2024-11-19T12:52:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T12:52:25,834 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 5461aaf01d3071b7aec04e96292d4b32 in 1382ms, sequenceid=237, compaction requested=false 2024-11-19T12:52:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T12:52:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T12:52:25,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T12:52:25,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7610 sec 2024-11-19T12:52:25,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.7650 sec 2024-11-19T12:52:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:26,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:26,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:26,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef58cb883c924039976baca4e7ebf2de_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:26,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742024_1200 (size=12404) 2024-11-19T12:52:26,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:26,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,124 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef58cb883c924039976baca4e7ebf2de_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef58cb883c924039976baca4e7ebf2de_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020806118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f41713ff63b475ab4682ca262d49f38, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f41713ff63b475ab4682ca262d49f38 is 175, key is test_row_0/A:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742025_1201 (size=31205) 2024-11-19T12:52:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020806119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020806119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020806119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020806231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020806241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020806241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020806241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020806441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020806451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020806451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020806451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,530 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f41713ff63b475ab4682ca262d49f38 2024-11-19T12:52:26,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/deb7a3c720504911867dead862230c39 is 50, key is test_row_0/B:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:26,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742026_1202 (size=12251) 2024-11-19T12:52:26,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/deb7a3c720504911867dead862230c39 2024-11-19T12:52:26,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/66f786e9572e44fcbc708aca5eb2e236 is 50, key is test_row_0/C:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:26,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742027_1203 (size=12251) 2024-11-19T12:52:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020806751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020806761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020806761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020806761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:26,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/66f786e9572e44fcbc708aca5eb2e236 2024-11-19T12:52:26,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/0f41713ff63b475ab4682ca262d49f38 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38 2024-11-19T12:52:26,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38, entries=150, sequenceid=260, filesize=30.5 K 2024-11-19T12:52:26,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/deb7a3c720504911867dead862230c39 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39 2024-11-19T12:52:26,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39, entries=150, sequenceid=260, filesize=12.0 K 2024-11-19T12:52:26,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/66f786e9572e44fcbc708aca5eb2e236 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236 2024-11-19T12:52:26,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236, entries=150, sequenceid=260, filesize=12.0 K 2024-11-19T12:52:26,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5461aaf01d3071b7aec04e96292d4b32 in 869ms, sequenceid=260, compaction requested=true 2024-11-19T12:52:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:26,974 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:26,974 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:26,975 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:26,975 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:26,975 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:26,975 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:26,975 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:26,975 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:26,975 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=91.7 K 2024-11-19T12:52:26,975 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f7000bfb17704148b09d772ac3fa289d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=36.2 K 2024-11-19T12:52:26,975 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38] 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f7000bfb17704148b09d772ac3fa289d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f834e1c81748c68ec34e0e924aede6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a5b97281f04a80b576e23c811afffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732020742769 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705d6c35a9794ebdaf2156212d764448, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732020742769 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f41713ff63b475ab4682ca262d49f38, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:26,976 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting deb7a3c720504911867dead862230c39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:26,983 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:26,985 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:26,985 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a0ca6062f9694ef9891bd407bd2b6da1_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:26,985 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/04cc58d90d464e6bb91748fc5866c86f is 50, key is test_row_0/B:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:26,987 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a0ca6062f9694ef9891bd407bd2b6da1_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:26,987 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a0ca6062f9694ef9891bd407bd2b6da1_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:26,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742029_1205 (size=4469) 2024-11-19T12:52:26,992 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#171 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:26,993 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/24927fc51ae34f4f8470831f986dc465 is 175, key is test_row_0/A:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742028_1204 (size=12865) 2024-11-19T12:52:27,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742030_1206 (size=31819) 2024-11-19T12:52:27,012 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/04cc58d90d464e6bb91748fc5866c86f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/04cc58d90d464e6bb91748fc5866c86f 2024-11-19T12:52:27,017 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into 04cc58d90d464e6bb91748fc5866c86f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:27,018 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:27,018 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=13, startTime=1732020746974; duration=0sec 2024-11-19T12:52:27,018 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:27,018 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:27,018 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:27,019 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:27,019 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:27,019 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,019 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/fb6246d1d9df4026987a69a8f8f1189d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=36.2 K 2024-11-19T12:52:27,020 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6246d1d9df4026987a69a8f8f1189d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732020741587 2024-11-19T12:52:27,022 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e6674485a9d74ce7812968f1f658e17b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732020742769 2024-11-19T12:52:27,023 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f786e9572e44fcbc708aca5eb2e236, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:27,031 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:27,032 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/6719af238d624c20ac91566bbb6cb8f6 is 50, key is test_row_0/C:col10/1732020746103/Put/seqid=0 2024-11-19T12:52:27,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742031_1207 (size=12865) 2024-11-19T12:52:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T12:52:27,241 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:27,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T12:52:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:27,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:27,244 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:27,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:27,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:27,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119351294363be440d3ae6e7f3048c4a5cd_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020747263/Put/seqid=0 2024-11-19T12:52:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742032_1208 (size=12454) 2024-11-19T12:52:27,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020807284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020807289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020807289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020807294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:27,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020807401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,414 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/24927fc51ae34f4f8470831f986dc465 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465 2024-11-19T12:52:27,421 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into 24927fc51ae34f4f8470831f986dc465(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:27,421 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:27,421 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=13, startTime=1732020746974; duration=0sec 2024-11-19T12:52:27,421 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:27,421 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:27,448 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/6719af238d624c20ac91566bbb6cb8f6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/6719af238d624c20ac91566bbb6cb8f6 2024-11-19T12:52:27,453 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into 6719af238d624c20ac91566bbb6cb8f6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:27,453 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:27,453 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=13, startTime=1732020746974; duration=0sec 2024-11-19T12:52:27,454 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:27,454 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:27,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020807611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020807611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020807611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020807611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:27,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119351294363be440d3ae6e7f3048c4a5cd_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119351294363be440d3ae6e7f3048c4a5cd_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:27,686 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/82000d574ad7451eaca4b7fe838a953e, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:27,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/82000d574ad7451eaca4b7fe838a953e is 175, key is test_row_0/A:col10/1732020747263/Put/seqid=0 2024-11-19T12:52:27,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742033_1209 (size=31255) 2024-11-19T12:52:27,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:27,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:27,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020807922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020807922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020807922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020807922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:27,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:28,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/82000d574ad7451eaca4b7fe838a953e 2024-11-19T12:52:28,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/00a966fc52464c96b15c4b3d570f0640 is 50, key is test_row_0/B:col10/1732020747263/Put/seqid=0 2024-11-19T12:52:28,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742034_1210 (size=12301) 2024-11-19T12:52:28,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020808432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:28,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:28,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020808432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020808432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020808441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:28,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/00a966fc52464c96b15c4b3d570f0640 2024-11-19T12:52:28,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/a63a9f7319b542ab9898f4198b7490a0 is 50, key is test_row_0/C:col10/1732020747263/Put/seqid=0 2024-11-19T12:52:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742035_1211 (size=12301) 2024-11-19T12:52:28,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/a63a9f7319b542ab9898f4198b7490a0 2024-11-19T12:52:28,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/82000d574ad7451eaca4b7fe838a953e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e 2024-11-19T12:52:28,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e, entries=150, sequenceid=275, filesize=30.5 K 2024-11-19T12:52:28,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:28,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/00a966fc52464c96b15c4b3d570f0640 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640 2024-11-19T12:52:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:28,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:28,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T12:52:28,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/a63a9f7319b542ab9898f4198b7490a0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0 2024-11-19T12:52:28,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T12:52:28,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5461aaf01d3071b7aec04e96292d4b32 in 1669ms, sequenceid=275, compaction requested=false 2024-11-19T12:52:28,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:29,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T12:52:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:29,076 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:52:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111935fad0d1f03641eebb23c2abe676f074_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020747288/Put/seqid=0 2024-11-19T12:52:29,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742036_1212 (size=12454) 2024-11-19T12:52:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:29,090 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111935fad0d1f03641eebb23c2abe676f074_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111935fad0d1f03641eebb23c2abe676f074_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:29,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/a15a120767924f118be383b6ab756d3e, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/a15a120767924f118be383b6ab756d3e is 175, key is test_row_0/A:col10/1732020747288/Put/seqid=0 2024-11-19T12:52:29,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742037_1213 (size=31255) 2024-11-19T12:52:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:29,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. as already flushing 2024-11-19T12:52:29,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020809462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020809463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020809465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020809466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,506 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/a15a120767924f118be383b6ab756d3e 2024-11-19T12:52:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b80cb11855f84400809a34bddad86af5 is 50, key is test_row_0/B:col10/1732020747288/Put/seqid=0 2024-11-19T12:52:29,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742038_1214 (size=12301) 2024-11-19T12:52:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020809572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020809572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020809572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020809572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020809782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020809782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020809782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020809782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:29,917 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b80cb11855f84400809a34bddad86af5 2024-11-19T12:52:29,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6a78eac6d7c4245aa0f77cdb8da5624 is 50, key is test_row_0/C:col10/1732020747288/Put/seqid=0 2024-11-19T12:52:29,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742039_1215 (size=12301) 2024-11-19T12:52:30,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34314 deadline: 1732020810091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34280 deadline: 1732020810092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34268 deadline: 1732020810092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34292 deadline: 1732020810092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:52:30,331 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6a78eac6d7c4245aa0f77cdb8da5624 2024-11-19T12:52:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/a15a120767924f118be383b6ab756d3e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e 2024-11-19T12:52:30,341 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e, entries=150, sequenceid=295, filesize=30.5 K 2024-11-19T12:52:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/b80cb11855f84400809a34bddad86af5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5 2024-11-19T12:52:30,346 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T12:52:30,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/e6a78eac6d7c4245aa0f77cdb8da5624 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624 2024-11-19T12:52:30,351 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T12:52:30,352 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5461aaf01d3071b7aec04e96292d4b32 in 1276ms, sequenceid=295, compaction requested=true 2024-11-19T12:52:30,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:30,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:30,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T12:52:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T12:52:30,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T12:52:30,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1090 sec 2024-11-19T12:52:30,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.1140 sec 2024-11-19T12:52:30,435 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-19T12:52:30,435 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,435 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,435 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,436 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-19T12:52:30,436 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,436 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,436 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,437 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-19T12:52:30,437 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,437 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,437 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,442 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-19T12:52:30,442 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,442 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,442 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:30,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:52:30,607 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-19T12:52:30,607 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-19T12:52:30,607 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,607 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,607 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:30,607 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,607 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-19T12:52:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:30,607 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,607 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,607 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:30,607 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:30,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:30,608 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,610 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-19T12:52:30,610 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:30,610 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,610 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:30,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:30,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c88a1ea9c95e4d22870d1742fd85ace5_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742040_1216 (size=12454) 2024-11-19T12:52:31,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:31,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c88a1ea9c95e4d22870d1742fd85ace5_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88a1ea9c95e4d22870d1742fd85ace5_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:31,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/87241af687904983bab973287bcdaa05, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:31,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/87241af687904983bab973287bcdaa05 is 175, key is test_row_0/A:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:31,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742041_1217 (size=31255) 2024-11-19T12:52:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T12:52:31,402 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:31,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/87241af687904983bab973287bcdaa05 2024-11-19T12:52:31,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ebb82b52e3bd43f898164952b7cedac5 is 50, key is test_row_0/B:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:31,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742042_1218 (size=12301) 2024-11-19T12:52:31,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ebb82b52e3bd43f898164952b7cedac5 2024-11-19T12:52:31,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/d7ce27b1cd8c48efbee0b7b18ecf5698 is 50, key is test_row_0/C:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:31,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742043_1219 (size=12301) 2024-11-19T12:52:32,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/d7ce27b1cd8c48efbee0b7b18ecf5698 2024-11-19T12:52:32,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/87241af687904983bab973287bcdaa05 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05 2024-11-19T12:52:32,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05, entries=150, sequenceid=312, filesize=30.5 K 2024-11-19T12:52:32,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/ebb82b52e3bd43f898164952b7cedac5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5 2024-11-19T12:52:32,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T12:52:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/d7ce27b1cd8c48efbee0b7b18ecf5698 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698 2024-11-19T12:52:32,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T12:52:32,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 5461aaf01d3071b7aec04e96292d4b32 in 1719ms, sequenceid=312, compaction requested=true 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5461aaf01d3071b7aec04e96292d4b32:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:32,325 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:32,325 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:32,326 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125584 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:32,326 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:32,326 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/B is initiating minor compaction (all files) 2024-11-19T12:52:32,326 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/A is initiating minor compaction (all files) 2024-11-19T12:52:32,326 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/B in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:32,326 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/A in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:32,327 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/04cc58d90d464e6bb91748fc5866c86f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=48.6 K 2024-11-19T12:52:32,327 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=122.6 K 2024-11-19T12:52:32,327 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:32,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05] 2024-11-19T12:52:32,327 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 04cc58d90d464e6bb91748fc5866c86f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:32,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24927fc51ae34f4f8470831f986dc465, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:32,327 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a966fc52464c96b15c4b3d570f0640, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020746116 2024-11-19T12:52:32,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82000d574ad7451eaca4b7fe838a953e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020746116 2024-11-19T12:52:32,328 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b80cb11855f84400809a34bddad86af5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732020747287 2024-11-19T12:52:32,328 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15a120767924f118be383b6ab756d3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732020747287 2024-11-19T12:52:32,328 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87241af687904983bab973287bcdaa05, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732020749454 2024-11-19T12:52:32,328 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb82b52e3bd43f898164952b7cedac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732020749454 2024-11-19T12:52:32,337 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:32,339 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ea18477e5aca41f4ba32d8669109b4f2_5461aaf01d3071b7aec04e96292d4b32 store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:32,339 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#B#compaction#182 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:32,340 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/0e9fbfee05f340ca9cc42a9652ab50c5 is 50, key is test_row_0/B:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:32,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742044_1220 (size=13051) 2024-11-19T12:52:32,364 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ea18477e5aca41f4ba32d8669109b4f2_5461aaf01d3071b7aec04e96292d4b32, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:32,365 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ea18477e5aca41f4ba32d8669109b4f2_5461aaf01d3071b7aec04e96292d4b32 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742045_1221 (size=4469) 2024-11-19T12:52:32,755 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/0e9fbfee05f340ca9cc42a9652ab50c5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/0e9fbfee05f340ca9cc42a9652ab50c5 2024-11-19T12:52:32,761 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/B of 5461aaf01d3071b7aec04e96292d4b32 into 0e9fbfee05f340ca9cc42a9652ab50c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:32,761 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:32,761 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/B, priority=12, startTime=1732020752325; duration=0sec 2024-11-19T12:52:32,761 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:32,761 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:B 2024-11-19T12:52:32,761 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:32,762 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:32,762 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 5461aaf01d3071b7aec04e96292d4b32/C is initiating minor compaction (all files) 2024-11-19T12:52:32,762 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5461aaf01d3071b7aec04e96292d4b32/C in TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:32,762 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/6719af238d624c20ac91566bbb6cb8f6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp, totalSize=48.6 K 2024-11-19T12:52:32,763 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6719af238d624c20ac91566bbb6cb8f6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732020744964 2024-11-19T12:52:32,763 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a63a9f7319b542ab9898f4198b7490a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732020746116 2024-11-19T12:52:32,763 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a78eac6d7c4245aa0f77cdb8da5624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732020747287 2024-11-19T12:52:32,764 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ce27b1cd8c48efbee0b7b18ecf5698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732020749454 2024-11-19T12:52:32,770 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:32,770 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/8a53d200c1e641b1bd96a950a29a4508 is 175, key is test_row_0/A:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:32,771 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5461aaf01d3071b7aec04e96292d4b32#C#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:32,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/9c7fa87236234f6894b318cbdd8e1670 is 50, key is test_row_0/C:col10/1732020750603/Put/seqid=0 2024-11-19T12:52:32,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742046_1222 (size=32005) 2024-11-19T12:52:32,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742047_1223 (size=13051) 2024-11-19T12:52:32,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:52:33,185 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/8a53d200c1e641b1bd96a950a29a4508 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/8a53d200c1e641b1bd96a950a29a4508 2024-11-19T12:52:33,186 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/9c7fa87236234f6894b318cbdd8e1670 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/9c7fa87236234f6894b318cbdd8e1670 2024-11-19T12:52:33,191 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/A of 5461aaf01d3071b7aec04e96292d4b32 into 8a53d200c1e641b1bd96a950a29a4508(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:33,191 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5461aaf01d3071b7aec04e96292d4b32/C of 5461aaf01d3071b7aec04e96292d4b32 into 9c7fa87236234f6894b318cbdd8e1670(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5461aaf01d3071b7aec04e96292d4b32: 2024-11-19T12:52:33,191 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/C, priority=12, startTime=1732020752325; duration=0sec 2024-11-19T12:52:33,191 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., storeName=5461aaf01d3071b7aec04e96292d4b32/A, priority=12, startTime=1732020752325; duration=0sec 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:C 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:33,191 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5461aaf01d3071b7aec04e96292d4b32:A 2024-11-19T12:52:34,390 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-19T12:52:34,390 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:34,390 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:34,390 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:34,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3479 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3519 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1468 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4404 rows 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1481 2024-11-19T12:52:34,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4437 rows 2024-11-19T12:52:34,391 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:52:34,391 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:52:34,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:34,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:34,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:34,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:52:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:34,394 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020754394"}]},"ts":"1732020754394"} 2024-11-19T12:52:34,396 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:52:34,396 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:52:34,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:52:34,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, UNASSIGN}] 2024-11-19T12:52:34,398 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, UNASSIGN 2024-11-19T12:52:34,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:34,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, UNASSIGN because future has completed 2024-11-19T12:52:34,400 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:52:34,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:34,553 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:34,553 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:52:34,554 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 5461aaf01d3071b7aec04e96292d4b32, disabling compactions & flushes 2024-11-19T12:52:34,554 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:34,554 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:34,554 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. after waiting 0 ms 2024-11-19T12:52:34,554 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:34,554 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 5461aaf01d3071b7aec04e96292d4b32 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T12:52:34,555 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=A 2024-11-19T12:52:34,555 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:34,555 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=B 2024-11-19T12:52:34,556 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:34,556 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5461aaf01d3071b7aec04e96292d4b32, store=C 2024-11-19T12:52:34,556 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:34,565 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e5a32ea880c4d6a882c18b16e104dd7_5461aaf01d3071b7aec04e96292d4b32 is 50, key is test_row_0/A:col10/1732020754373/Put/seqid=0 2024-11-19T12:52:34,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742048_1224 (size=9914) 2024-11-19T12:52:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:34,971 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:34,982 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e5a32ea880c4d6a882c18b16e104dd7_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e5a32ea880c4d6a882c18b16e104dd7_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:34,983 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b744c787194e4ea4879abdcaec0a0794, store: [table=TestAcidGuarantees family=A region=5461aaf01d3071b7aec04e96292d4b32] 2024-11-19T12:52:34,984 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b744c787194e4ea4879abdcaec0a0794 is 175, key is test_row_0/A:col10/1732020754373/Put/seqid=0 2024-11-19T12:52:34,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742049_1225 (size=22561) 2024-11-19T12:52:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:35,389 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b744c787194e4ea4879abdcaec0a0794 2024-11-19T12:52:35,406 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/5882abc75ef045afb4893c31f8c4fdc8 is 50, key is test_row_0/B:col10/1732020754373/Put/seqid=0 2024-11-19T12:52:35,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742050_1226 (size=9857) 2024-11-19T12:52:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:35,811 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/5882abc75ef045afb4893c31f8c4fdc8 2024-11-19T12:52:35,826 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/5f1e69a974ea423a9728cb1a17758542 is 50, key is test_row_0/C:col10/1732020754373/Put/seqid=0 2024-11-19T12:52:35,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742051_1227 (size=9857) 2024-11-19T12:52:36,232 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/5f1e69a974ea423a9728cb1a17758542 2024-11-19T12:52:36,272 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/A/b744c787194e4ea4879abdcaec0a0794 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b744c787194e4ea4879abdcaec0a0794 2024-11-19T12:52:36,279 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b744c787194e4ea4879abdcaec0a0794, entries=100, sequenceid=320, filesize=22.0 K 2024-11-19T12:52:36,280 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/B/5882abc75ef045afb4893c31f8c4fdc8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/5882abc75ef045afb4893c31f8c4fdc8 2024-11-19T12:52:36,284 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/5882abc75ef045afb4893c31f8c4fdc8, entries=100, sequenceid=320, filesize=9.6 K 2024-11-19T12:52:36,284 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/.tmp/C/5f1e69a974ea423a9728cb1a17758542 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/5f1e69a974ea423a9728cb1a17758542 2024-11-19T12:52:36,288 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/5f1e69a974ea423a9728cb1a17758542, entries=100, sequenceid=320, filesize=9.6 K 2024-11-19T12:52:36,289 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 5461aaf01d3071b7aec04e96292d4b32 in 1735ms, sequenceid=320, compaction requested=false 2024-11-19T12:52:36,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05] to archive 2024-11-19T12:52:36,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:36,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/2b9aa7d1157a446fb6176f5f8d008704 2024-11-19T12:52:36,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/4640f00ec58747dcaaf144141df7a149 2024-11-19T12:52:36,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/7834a0dd78134addb0c03e73742a5088 2024-11-19T12:52:36,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/11a223ba845541438c1fc9fb9bf0ed57 2024-11-19T12:52:36,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/56b9c7b6491b48d48522745201382bf0 2024-11-19T12:52:36,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/923c58e494b54ad9820b7f8d9ef8b037 2024-11-19T12:52:36,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d6eb5e8e778f4299afea19cedcb86a49 2024-11-19T12:52:36,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/96eaa8bc38ee4719b780e20d6fa95a03 2024-11-19T12:52:36,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/c5ad0e1c36b343aea6732b22379a6ab9 2024-11-19T12:52:36,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/d401ba41640b4c108655a0a0a57289a0 2024-11-19T12:52:36,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/623da7f2149443e1812c0234a793f9a1 2024-11-19T12:52:36,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/1d24ec6e3a5241d5a33761d47b7ec492 2024-11-19T12:52:36,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b348a2943c0c42099112bc3b3da1a34c 2024-11-19T12:52:36,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/260a0a3dbee74ff1861cab6d251e0e9a 2024-11-19T12:52:36,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f21177f4a9545efb52ca1554765a90c 2024-11-19T12:52:36,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/e1f834e1c81748c68ec34e0e924aede6 2024-11-19T12:52:36,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/705d6c35a9794ebdaf2156212d764448 2024-11-19T12:52:36,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/24927fc51ae34f4f8470831f986dc465 2024-11-19T12:52:36,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/0f41713ff63b475ab4682ca262d49f38 2024-11-19T12:52:36,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/82000d574ad7451eaca4b7fe838a953e 2024-11-19T12:52:36,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/a15a120767924f118be383b6ab756d3e 2024-11-19T12:52:36,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/87241af687904983bab973287bcdaa05 2024-11-19T12:52:36,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/366790f02b6543809aecb70c3eac9b67, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/419efda2eb024e6fb4f7b20e86b5b317, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/dfec101f76c6496491611074c8455a94, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1103bee56e344ab9afdf7cd91e5feeca, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f7000bfb17704148b09d772ac3fa289d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/04cc58d90d464e6bb91748fc5866c86f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5] to archive 2024-11-19T12:52:36,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/4ed5c764955945a1af1c27f9f61319b7 2024-11-19T12:52:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/72e6b36a0c4940eea258436d25cbd979 2024-11-19T12:52:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/df11648bf05f4d3b9f5d6f0f54f496b9 2024-11-19T12:52:36,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/366790f02b6543809aecb70c3eac9b67 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/366790f02b6543809aecb70c3eac9b67 2024-11-19T12:52:36,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1a3b570781ea436daacb6b6c51f71005 2024-11-19T12:52:36,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/419efda2eb024e6fb4f7b20e86b5b317 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/419efda2eb024e6fb4f7b20e86b5b317 2024-11-19T12:52:36,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b055ca2372f64d289e0c1b4e08a69310 2024-11-19T12:52:36,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/207ba886981346ae8935cb3581c9906e 2024-11-19T12:52:36,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/dfec101f76c6496491611074c8455a94 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/dfec101f76c6496491611074c8455a94 2024-11-19T12:52:36,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f04c4fc29273496a91a4f753e4453cb0 2024-11-19T12:52:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ae17fb105dd24ca0b67bb6ad724b7824 2024-11-19T12:52:36,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1103bee56e344ab9afdf7cd91e5feeca to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/1103bee56e344ab9afdf7cd91e5feeca 2024-11-19T12:52:36,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/17aefc8ba6884255a2b749c114c32804 2024-11-19T12:52:36,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/c5e3cc967bca4f4ebb1bb142d4a83c15 2024-11-19T12:52:36,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f7000bfb17704148b09d772ac3fa289d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/f7000bfb17704148b09d772ac3fa289d 2024-11-19T12:52:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/e8c2bd1171de463b8878ab647e9f2294 2024-11-19T12:52:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/05a5b97281f04a80b576e23c811afffc 2024-11-19T12:52:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/04cc58d90d464e6bb91748fc5866c86f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/04cc58d90d464e6bb91748fc5866c86f 2024-11-19T12:52:36,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/deb7a3c720504911867dead862230c39 2024-11-19T12:52:36,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/00a966fc52464c96b15c4b3d570f0640 2024-11-19T12:52:36,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/b80cb11855f84400809a34bddad86af5 2024-11-19T12:52:36,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/ebb82b52e3bd43f898164952b7cedac5 2024-11-19T12:52:36,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/c904e1948f544b0dad71917be5e72dba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/dda9ac4d8fc44b48907ecef9f08208b6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/23d6ddbd072c4187ba242c4a4253f1c5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3bbc171c81cf430c86a643c710851965, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/fb6246d1d9df4026987a69a8f8f1189d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/6719af238d624c20ac91566bbb6cb8f6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698] to archive 2024-11-19T12:52:36,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:52:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3d399d81a84f450892b5d74f635b6256 2024-11-19T12:52:36,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/bf5ff2e945cc454fa5e4826dcc5626f5 2024-11-19T12:52:36,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/c904e1948f544b0dad71917be5e72dba to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/c904e1948f544b0dad71917be5e72dba 2024-11-19T12:52:36,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/f84cbc6017b74466ac6db274820d56f9 2024-11-19T12:52:36,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/87dde872f0d34fa1959cac9cbf118fb4 2024-11-19T12:52:36,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/dda9ac4d8fc44b48907ecef9f08208b6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/dda9ac4d8fc44b48907ecef9f08208b6 2024-11-19T12:52:36,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2e439cd3467e4daeb030420d291f0cf4 2024-11-19T12:52:36,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/76b8cc72ecb147c19e80e2ce9405f3b2 2024-11-19T12:52:36,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/23d6ddbd072c4187ba242c4a4253f1c5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/23d6ddbd072c4187ba242c4a4253f1c5 2024-11-19T12:52:36,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/7331f460c8034df88c264a9cd4db1675 2024-11-19T12:52:36,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/aa656ea4c24042bc9f9dd3df0f8392d4 2024-11-19T12:52:36,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3bbc171c81cf430c86a643c710851965 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/3bbc171c81cf430c86a643c710851965 2024-11-19T12:52:36,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/2242af58acc4482493ec89b2cce3af6b 2024-11-19T12:52:36,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/00cb988a0565475db8e557b4a716d2f3 2024-11-19T12:52:36,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/fb6246d1d9df4026987a69a8f8f1189d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/fb6246d1d9df4026987a69a8f8f1189d 2024-11-19T12:52:36,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/4a4251efe25f445cb831043c96ab2d89 2024-11-19T12:52:36,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6674485a9d74ce7812968f1f658e17b 2024-11-19T12:52:36,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/6719af238d624c20ac91566bbb6cb8f6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/6719af238d624c20ac91566bbb6cb8f6 2024-11-19T12:52:36,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/66f786e9572e44fcbc708aca5eb2e236 2024-11-19T12:52:36,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/a63a9f7319b542ab9898f4198b7490a0 2024-11-19T12:52:36,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/e6a78eac6d7c4245aa0f77cdb8da5624 2024-11-19T12:52:36,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/d7ce27b1cd8c48efbee0b7b18ecf5698 2024-11-19T12:52:36,359 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits/323.seqid, newMaxSeqId=323, maxSeqId=4 2024-11-19T12:52:36,360 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. 2024-11-19T12:52:36,360 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 5461aaf01d3071b7aec04e96292d4b32: Waiting for close lock at 1732020754553Running coprocessor pre-close hooks at 1732020754553Disabling compacts and flushes for region at 1732020754553Disabling writes for close at 1732020754554 (+1 ms)Obtaining lock to block concurrent updates at 1732020754554Preparing flush snapshotting stores in 5461aaf01d3071b7aec04e96292d4b32 at 1732020754554Finished memstore snapshotting TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732020754556 (+2 ms)Flushing stores of TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32. at 1732020754557 (+1 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/A: creating writer at 1732020754558 (+1 ms)Flushing mob file 5461aaf01d3071b7aec04e96292d4b32/A: appending metadata at 1732020754565 (+7 ms)Flushing mob file 5461aaf01d3071b7aec04e96292d4b32/A: closing flushed file at 1732020754565Flushing 5461aaf01d3071b7aec04e96292d4b32/A: appending metadata at 1732020754983 (+418 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/A: closing flushed file at 1732020754984 (+1 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/B: creating writer at 1732020755400 (+416 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/B: appending metadata at 1732020755405 (+5 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/B: closing flushed file at 1732020755405Flushing 5461aaf01d3071b7aec04e96292d4b32/C: creating writer at 1732020755821 (+416 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/C: appending metadata at 1732020755825 (+4 ms)Flushing 5461aaf01d3071b7aec04e96292d4b32/C: closing flushed file at 1732020755825Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26986ba5: reopening flushed file at 1732020756271 (+446 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5effd7ad: reopening flushed file at 1732020756279 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bd55221: reopening flushed file at 1732020756284 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 5461aaf01d3071b7aec04e96292d4b32 in 1735ms, sequenceid=320, compaction requested=false at 1732020756289 (+5 ms)Writing region close event to WAL at 1732020756356 (+67 ms)Running coprocessor post-close hooks at 1732020756360 (+4 ms)Closed at 1732020756360 2024-11-19T12:52:36,362 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,362 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=5461aaf01d3071b7aec04e96292d4b32, regionState=CLOSED 2024-11-19T12:52:36,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:36,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T12:52:36,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 5461aaf01d3071b7aec04e96292d4b32, server=aba5a916dfea,35349,1732020694570 in 1.9640 sec 2024-11-19T12:52:36,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-19T12:52:36,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5461aaf01d3071b7aec04e96292d4b32, UNASSIGN in 1.9690 sec 2024-11-19T12:52:36,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T12:52:36,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9720 sec 2024-11-19T12:52:36,371 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020756370"}]},"ts":"1732020756370"} 2024-11-19T12:52:36,372 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:52:36,372 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:52:36,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9800 sec 2024-11-19T12:52:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T12:52:36,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:36,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:52:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,545 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T12:52:36,546 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,551 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,556 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits] 2024-11-19T12:52:36,563 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/8a53d200c1e641b1bd96a950a29a4508 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/8a53d200c1e641b1bd96a950a29a4508 2024-11-19T12:52:36,567 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b744c787194e4ea4879abdcaec0a0794 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/A/b744c787194e4ea4879abdcaec0a0794 2024-11-19T12:52:36,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/0e9fbfee05f340ca9cc42a9652ab50c5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/0e9fbfee05f340ca9cc42a9652ab50c5 2024-11-19T12:52:36,577 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/5882abc75ef045afb4893c31f8c4fdc8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/B/5882abc75ef045afb4893c31f8c4fdc8 2024-11-19T12:52:36,581 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/5f1e69a974ea423a9728cb1a17758542 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/5f1e69a974ea423a9728cb1a17758542 2024-11-19T12:52:36,582 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/9c7fa87236234f6894b318cbdd8e1670 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/C/9c7fa87236234f6894b318cbdd8e1670 2024-11-19T12:52:36,585 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits/323.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32/recovered.edits/323.seqid 2024-11-19T12:52:36,586 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,586 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:52:36,587 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:52:36,588 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T12:52:36,592 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119088f5e3e4a0e499189dfea2f21a83ec8_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119088f5e3e4a0e499189dfea2f21a83ec8_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,593 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b01727e61944e249d37cd24f62102ab_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b01727e61944e249d37cd24f62102ab_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,594 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912a10b72e3084a6f897d5295827f65e0_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912a10b72e3084a6f897d5295827f65e0_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,595 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111913124cc90e754680b82f009dc9dc7767_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111913124cc90e754680b82f009dc9dc7767_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,596 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922ab113a741245a3b3e92d9fbba50947_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922ab113a741245a3b3e92d9fbba50947_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,598 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923568365b7f148d598a297186658bd1d_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923568365b7f148d598a297186658bd1d_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,599 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192917be66880f48608e3244d4d433996c_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192917be66880f48608e3244d4d433996c_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,599 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119351294363be440d3ae6e7f3048c4a5cd_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119351294363be440d3ae6e7f3048c4a5cd_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,600 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111935fad0d1f03641eebb23c2abe676f074_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111935fad0d1f03641eebb23c2abe676f074_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,601 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119671cb05a4140451a81a5392044e0c241_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119671cb05a4140451a81a5392044e0c241_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,602 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e5a32ea880c4d6a882c18b16e104dd7_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e5a32ea880c4d6a882c18b16e104dd7_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adb6333a42834d0684b1961a7016602c_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adb6333a42834d0684b1961a7016602c_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,604 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9c928cabbd54d3a805808037d6b394e_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9c928cabbd54d3a805808037d6b394e_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,605 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0d847dab2f24074af990c99592b4843_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0d847dab2f24074af990c99592b4843_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c260bf9ebce34532bd25418300494037_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c260bf9ebce34532bd25418300494037_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,607 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88a1ea9c95e4d22870d1742fd85ace5_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c88a1ea9c95e4d22870d1742fd85ace5_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,608 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef58cb883c924039976baca4e7ebf2de_5461aaf01d3071b7aec04e96292d4b32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef58cb883c924039976baca4e7ebf2de_5461aaf01d3071b7aec04e96292d4b32 2024-11-19T12:52:36,609 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:52:36,611 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,613 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:52:36,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:52:36,617 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,617 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:52:36,617 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020756617"}]},"ts":"9223372036854775807"} 2024-11-19T12:52:36,619 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:52:36,619 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5461aaf01d3071b7aec04e96292d4b32, NAME => 'TestAcidGuarantees,,1732020727239.5461aaf01d3071b7aec04e96292d4b32.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:52:36,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:52:36,619 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020756619"}]},"ts":"9223372036854775807"} 2024-11-19T12:52:36,621 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:52:36,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-19T12:52:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T12:52:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:52:36,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:36,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-554143411_22 at /127.0.0.1:60808 [Waiting for operation #194] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2036620531_22 at /127.0.0.1:60810 [Waiting for operation #195] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-554143411_22 at /127.0.0.1:60770 [Waiting for operation #212] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2036620531_22 at /127.0.0.1:60692 [Waiting for operation #239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=5812 (was 5871) 2024-11-19T12:52:36,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=5811 2024-11-19T12:52:36,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:52:36,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:52:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:52:36,673 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:52:36,673 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:36,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-19T12:52:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:36,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:52:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742052_1228 (size=1061) 2024-11-19T12:52:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:37,082 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:52:37,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742053_1229 (size=53) 2024-11-19T12:52:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:37,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:52:37,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8451cbb661c52001cbf3306ac11815b1, disabling compactions & flushes 2024-11-19T12:52:37,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. after waiting 0 ms 2024-11-19T12:52:37,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,493 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8451cbb661c52001cbf3306ac11815b1: Waiting for close lock at 1732020757492Disabling compacts and flushes for region at 1732020757492Disabling writes for close at 1732020757493 (+1 ms)Writing region close event to WAL at 1732020757493Closed at 1732020757493 2024-11-19T12:52:37,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:52:37,496 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020757495"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020757495"}]},"ts":"1732020757495"} 2024-11-19T12:52:37,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:52:37,501 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:52:37,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020757501"}]},"ts":"1732020757501"} 2024-11-19T12:52:37,505 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:52:37,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, ASSIGN}] 2024-11-19T12:52:37,507 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, ASSIGN 2024-11-19T12:52:37,508 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:52:37,659 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8451cbb661c52001cbf3306ac11815b1, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:37,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, ASSIGN because future has completed 2024-11-19T12:52:37,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:52:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:37,824 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,825 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:52:37,825 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,825 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:52:37,826 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,826 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,828 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,830 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:37,831 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8451cbb661c52001cbf3306ac11815b1 columnFamilyName A 2024-11-19T12:52:37,831 DEBUG [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:37,832 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(327): Store=8451cbb661c52001cbf3306ac11815b1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:37,832 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,834 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:37,834 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8451cbb661c52001cbf3306ac11815b1 columnFamilyName B 2024-11-19T12:52:37,834 DEBUG [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:37,834 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(327): Store=8451cbb661c52001cbf3306ac11815b1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:37,835 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,836 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:52:37,836 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8451cbb661c52001cbf3306ac11815b1 columnFamilyName C 2024-11-19T12:52:37,836 DEBUG [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:52:37,837 INFO [StoreOpener-8451cbb661c52001cbf3306ac11815b1-1 {}] regionserver.HStore(327): Store=8451cbb661c52001cbf3306ac11815b1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:52:37,837 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,837 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,837 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,838 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:52:37,840 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,842 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:52:37,842 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 8451cbb661c52001cbf3306ac11815b1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62768003, jitterRate=-0.06468386948108673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:52:37,843 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:37,844 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 8451cbb661c52001cbf3306ac11815b1: Running coprocessor pre-open hook at 1732020757826Writing region info on filesystem at 1732020757826Initializing all the Stores at 1732020757827 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020757828 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020757828Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020757828Cleaning up temporary data from old regions at 1732020757839 (+11 ms)Running coprocessor post-open hooks at 1732020757843 (+4 ms)Region opened successfully at 1732020757843 2024-11-19T12:52:37,844 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., pid=59, masterSystemTime=1732020757818 2024-11-19T12:52:37,846 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,846 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:37,846 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8451cbb661c52001cbf3306ac11815b1, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:37,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:52:37,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T12:52:37,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 in 184 msec 2024-11-19T12:52:37,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T12:52:37,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, ASSIGN in 345 msec 2024-11-19T12:52:37,853 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:52:37,854 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020757854"}]},"ts":"1732020757854"} 2024-11-19T12:52:37,855 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:52:37,856 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:52:37,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-19T12:52:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-19T12:52:38,832 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:38,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,836 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71daafb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a73d3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,839 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:52:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:52:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55a57c12 2024-11-19T12:52:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:52:38,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:52:38,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26acd927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a078a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,855 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e62be7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,858 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577af21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,861 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ca027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,862 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50dedd17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,864 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ee9712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,865 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef18675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cfa80b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de739e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35522cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a194a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d93706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206a7f1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719a5e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6253578a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492ae89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54706702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:52:38,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:52:38,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619aa4a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:52:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:38,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:52:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b214ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:52:38,884 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:38,884 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,884 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-19T12:52:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,886 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T12:52:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,887 DEBUG [Thread-1081 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,887 DEBUG [Thread-1081 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,887 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,891 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:38,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:38,893 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:52:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:52:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:52:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:38,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:52:38,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:52:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:38,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/877263a981254a06b13066a7eeb5e86f is 50, key is test_row_0/A:col10/1732020758895/Put/seqid=0 2024-11-19T12:52:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020818928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020818930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020758903, max timestamp=1732020758935Num uniques 859; 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020758903, max timestamp=1732020758935Num uniques 859; 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020758903, max timestamp=1732020758935Num uniques 859; 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:38,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:38,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742054_1230 (size=12001) 2024-11-19T12:52:38,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/877263a981254a06b13066a7eeb5e86f 2024-11-19T12:52:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020818970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020818972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020818976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T12:52:38,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d3b127db58843d28488a092fad8566c is 50, key is test_row_0/B:col10/1732020758895/Put/seqid=0 2024-11-19T12:52:39,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742055_1231 (size=12001) 2024-11-19T12:52:39,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d3b127db58843d28488a092fad8566c 2024-11-19T12:52:39,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020819041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020819041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T12:52:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:39,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:39,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/92665b922e524768aa946b6ab60a5fe9 is 50, key is test_row_0/C:col10/1732020758895/Put/seqid=0 2024-11-19T12:52:39,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742056_1232 (size=12001) 2024-11-19T12:52:39,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/92665b922e524768aa946b6ab60a5fe9 2024-11-19T12:52:39,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/877263a981254a06b13066a7eeb5e86f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f 2024-11-19T12:52:39,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T12:52:39,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d3b127db58843d28488a092fad8566c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c 2024-11-19T12:52:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020819081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020819081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T12:52:39,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020819091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/92665b922e524768aa946b6ab60a5fe9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9 2024-11-19T12:52:39,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T12:52:39,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8451cbb661c52001cbf3306ac11815b1 in 208ms, sequenceid=12, compaction requested=false 2024-11-19T12:52:39,104 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T12:52:39,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T12:52:39,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T12:52:39,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:39,202 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T12:52:39,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/c8642ba7195e4563be11093ebe6e46b8 is 50, key is test_row_0/A:col10/1732020758970/Put/seqid=0 2024-11-19T12:52:39,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742057_1233 (size=12001) 2024-11-19T12:52:39,237 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/c8642ba7195e4563be11093ebe6e46b8 2024-11-19T12:52:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/19a4d6aace324970b2eec654ab240ba5 is 50, key is test_row_0/B:col10/1732020758970/Put/seqid=0 2024-11-19T12:52:39,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742058_1234 (size=12001) 2024-11-19T12:52:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020819273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020819272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020819291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020819291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020819301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020819381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020819381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T12:52:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020819591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020819591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020819602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020819602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020819611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,653 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/19a4d6aace324970b2eec654ab240ba5 2024-11-19T12:52:39,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/95d97e5008124c1bbe34e7a4a11f30d8 is 50, key is test_row_0/C:col10/1732020758970/Put/seqid=0 2024-11-19T12:52:39,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742059_1235 (size=12001) 2024-11-19T12:52:39,681 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/95d97e5008124c1bbe34e7a4a11f30d8 2024-11-19T12:52:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/c8642ba7195e4563be11093ebe6e46b8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8 2024-11-19T12:52:39,689 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T12:52:39,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/19a4d6aace324970b2eec654ab240ba5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5 2024-11-19T12:52:39,694 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T12:52:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/95d97e5008124c1bbe34e7a4a11f30d8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8 2024-11-19T12:52:39,699 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T12:52:39,700 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 8451cbb661c52001cbf3306ac11815b1 in 498ms, sequenceid=48, compaction requested=false 2024-11-19T12:52:39,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:39,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:39,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-19T12:52:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-19T12:52:39,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T12:52:39,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-11-19T12:52:39,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 820 msec 2024-11-19T12:52:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:39,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:52:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:39,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:39,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:39,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:39,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/424d9247cce343b78bb3f4ddabf50b8a is 50, key is test_row_0/A:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:39,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020819938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742060_1236 (size=14341) 2024-11-19T12:52:39,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/424d9247cce343b78bb3f4ddabf50b8a 2024-11-19T12:52:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020819942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:39,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/050ba5ef8f3c4ff0aa912475bce5ce2e is 50, key is test_row_0/B:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:39,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742061_1237 (size=12001) 2024-11-19T12:52:39,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/050ba5ef8f3c4ff0aa912475bce5ce2e 2024-11-19T12:52:39,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/7f0b078b720b448fb3c24c71daad7114 is 50, key is test_row_0/C:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:39,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742062_1238 (size=12001) 2024-11-19T12:52:39,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/7f0b078b720b448fb3c24c71daad7114 2024-11-19T12:52:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/424d9247cce343b78bb3f4ddabf50b8a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a 2024-11-19T12:52:39,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a, entries=200, sequenceid=61, filesize=14.0 K 2024-11-19T12:52:39,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/050ba5ef8f3c4ff0aa912475bce5ce2e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e 2024-11-19T12:52:39,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T12:52:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/7f0b078b720b448fb3c24c71daad7114 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114 2024-11-19T12:52:40,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T12:52:40,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8451cbb661c52001cbf3306ac11815b1 in 100ms, sequenceid=61, compaction requested=true 2024-11-19T12:52:40,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:40,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:40,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:40,004 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:40,004 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:40,007 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,007 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,007 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=37.4 K 2024-11-19T12:52:40,007 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=35.2 K 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3b127db58843d28488a092fad8566c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732020758892 2024-11-19T12:52:40,007 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 877263a981254a06b13066a7eeb5e86f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732020758892 2024-11-19T12:52:40,008 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a4d6aace324970b2eec654ab240ba5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732020758968 2024-11-19T12:52:40,008 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8642ba7195e4563be11093ebe6e46b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732020758968 2024-11-19T12:52:40,008 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424d9247cce343b78bb3f4ddabf50b8a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:40,008 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 050ba5ef8f3c4ff0aa912475bce5ce2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:40,018 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:40,018 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/f330baadcde540f48c4e7de4c7d07e98 is 50, key is test_row_0/B:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:40,019 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:40,020 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/807b332108234c639fdcc75b00b89202 is 50, key is test_row_0/A:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T12:52:40,021 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:40,022 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T12:52:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:40,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:40,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:40,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742064_1240 (size=12104) 2024-11-19T12:52:40,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742063_1239 (size=12104) 2024-11-19T12:52:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:40,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732020759930, max timestamp=1732020760054Num uniques 859; 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732020759930, max timestamp=1732020760054Num uniques 859; 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732020759930, max timestamp=1732020760054Num uniques 859; 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:40,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:40,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/135b8955db7f4b9383962ddd503cb9b8 is 50, key is test_row_0/A:col10/1732020759940/Put/seqid=0 2024-11-19T12:52:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742065_1241 (size=16681) 2024-11-19T12:52:40,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020760089, max timestamp=1732020760110Num uniques 859; 2024-11-19T12:52:40,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020760102, max timestamp=1732020760110Num uniques 859; 2024-11-19T12:52:40,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:40,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:40,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020760089, max timestamp=1732020760110Num uniques 859; 2024-11-19T12:52:40,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:40,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020820132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732020760128, max timestamp=1732020760135Num uniques 1068; 2024-11-19T12:52:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732020760128, max timestamp=1732020760135Num uniques 1068; 2024-11-19T12:52:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732020760128, max timestamp=1732020760135Num uniques 1018; 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T12:52:40,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T12:52:40,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020760157, max timestamp=1732020760164Num uniques 1027; 2024-11-19T12:52:40,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T12:52:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020760157, max timestamp=1732020760164Num uniques 1027; 2024-11-19T12:52:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T12:52:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020760157, max timestamp=1732020760164Num uniques 1027; 2024-11-19T12:52:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T12:52:40,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020820185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020760182, max timestamp=1732020760189Num uniques 1086; 2024-11-19T12:52:40,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T12:52:40,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020760182, max timestamp=1732020760189Num uniques 1086; 2024-11-19T12:52:40,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T12:52:40,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020760182, max timestamp=1732020760189Num uniques 1086; 2024-11-19T12:52:40,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T12:52:40,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020760218, max timestamp=1732020760226Num uniques 1095; 2024-11-19T12:52:40,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:52:40,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020760218, max timestamp=1732020760226Num uniques 1095; 2024-11-19T12:52:40,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020760218, max timestamp=1732020760226Num uniques 1095; 2024-11-19T12:52:40,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:52:40,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732020760228, max timestamp=1732020760297Num uniques 1054; 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732020760228, max timestamp=1732020760297Num uniques 1054; 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732020760228, max timestamp=1732020760297Num uniques 1054; 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T12:52:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T12:52:40,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732020760320, max timestamp=1732020760330Num uniques 1063; 2024-11-19T12:52:40,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T12:52:40,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732020760320, max timestamp=1732020760330Num uniques 1063; 2024-11-19T12:52:40,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T12:52:40,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732020760320, max timestamp=1732020760330Num uniques 1063; 2024-11-19T12:52:40,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=6870, suffix heap size=138520,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=150 2024-11-19T12:52:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:40,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732020760353, max timestamp=1732020760360Num uniques 1072; 2024-11-19T12:52:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732020760353, max timestamp=1732020760360Num uniques 1072; 2024-11-19T12:52:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T12:52:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T12:52:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732020760353, max timestamp=1732020760360Num uniques 1009; 2024-11-19T12:52:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10162, suffix heap size=132904,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=222 2024-11-19T12:52:40,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020820379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732020760370, max timestamp=1732020760385Num uniques 1081; 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732020760370, max timestamp=1732020760385Num uniques 1081; 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732020760370, max timestamp=1732020760385Num uniques 1081; 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-19T12:52:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T12:52:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732020760405, max timestamp=1732020760413Num uniques 1090; 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732020760405, max timestamp=1732020760413Num uniques 1040; 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732020760405, max timestamp=1732020760413Num uniques 1090; 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T12:52:40,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-19T12:52:40,431 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/807b332108234c639fdcc75b00b89202 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/807b332108234c639fdcc75b00b89202 2024-11-19T12:52:40,431 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/f330baadcde540f48c4e7de4c7d07e98 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/f330baadcde540f48c4e7de4c7d07e98 2024-11-19T12:52:40,436 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into f330baadcde540f48c4e7de4c7d07e98(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:40,436 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 807b332108234c639fdcc75b00b89202(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:40,436 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:40,436 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:40,436 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020760004; duration=0sec 2024-11-19T12:52:40,436 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020760004; duration=0sec 2024-11-19T12:52:40,436 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:40,436 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:40,436 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:40,437 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:40,437 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:40,438 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:40,438 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:40,438 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,438 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=35.2 K 2024-11-19T12:52:40,439 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 92665b922e524768aa946b6ab60a5fe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732020758892 2024-11-19T12:52:40,439 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d97e5008124c1bbe34e7a4a11f30d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732020758968 2024-11-19T12:52:40,440 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0b078b720b448fb3c24c71daad7114, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732020760424, max timestamp=1732020760443Num uniques 1099; 2024-11-19T12:52:40,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-19T12:52:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732020760424, max timestamp=1732020760443Num uniques 1099; 2024-11-19T12:52:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-19T12:52:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732020760438, max timestamp=1732020760443Num uniques 1049; 2024-11-19T12:52:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T12:52:40,453 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:40,454 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/e75816481af14b7db7f4a7a609a227b3 is 50, key is test_row_0/C:col10/1732020759271/Put/seqid=0 2024-11-19T12:52:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742066_1242 (size=12104) 2024-11-19T12:52:40,476 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/e75816481af14b7db7f4a7a609a227b3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e75816481af14b7db7f4a7a609a227b3 2024-11-19T12:52:40,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/135b8955db7f4b9383962ddd503cb9b8 2024-11-19T12:52:40,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2bc5467872b04e2bb7d15f3f9e2a53d7 is 50, key is test_row_0/B:col10/1732020759940/Put/seqid=0 2024-11-19T12:52:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742067_1243 (size=12001) 2024-11-19T12:52:40,513 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into e75816481af14b7db7f4a7a609a227b3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:40,513 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:40,513 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020760005; duration=0sec 2024-11-19T12:52:40,513 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:40,513 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732020760519, max timestamp=1732020760524Num uniques 1058; 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732020760519, max timestamp=1732020760524Num uniques 1158; 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1732020760493, max timestamp=1732020760524Num uniques 1158; 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-19T12:52:40,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=11810, suffix heap size=148400,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=258 2024-11-19T12:52:40,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732020760538, max timestamp=1732020760554Num uniques 1017; 2024-11-19T12:52:40,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1732020760538, max timestamp=1732020760554Num uniques 1117; 2024-11-19T12:52:40,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732020760538, max timestamp=1732020760554Num uniques 1017; 2024-11-19T12:52:40,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=12222, suffix heap size=144136,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=267 2024-11-19T12:52:40,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=6870, suffix heap size=133736,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=150 2024-11-19T12:52:40,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T12:52:40,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732020760568, max timestamp=1732020760580Num uniques 1126; 2024-11-19T12:52:40,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=12636, suffix heap size=145072,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=276 2024-11-19T12:52:40,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732020760568, max timestamp=1732020760580Num uniques 1076; 2024-11-19T12:52:40,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-19T12:52:40,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732020760568, max timestamp=1732020760580Num uniques 1009; 2024-11-19T12:52:40,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12636, suffix heap size=132904,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=276 2024-11-19T12:52:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020820596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020820596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020820597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732020760593, max timestamp=1732020760600Num uniques 1135; 2024-11-19T12:52:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732020760593, max timestamp=1732020760600Num uniques 1135; 2024-11-19T12:52:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-19T12:52:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-19T12:52:40,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732020760593, max timestamp=1732020760600Num uniques 1135; 2024-11-19T12:52:40,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-19T12:52:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732020760620, max timestamp=1732020760640Num uniques 1094; 2024-11-19T12:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732020760620, max timestamp=1732020760640Num uniques 1094; 2024-11-19T12:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-19T12:52:40,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-19T12:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732020760620, max timestamp=1732020760640Num uniques 1094; 2024-11-19T12:52:40,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-19T12:52:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:40,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732020760667, max timestamp=1732020760684Num uniques 1053; 2024-11-19T12:52:40,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-19T12:52:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732020760667, max timestamp=1732020760684Num uniques 1053; 2024-11-19T12:52:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732020760667, max timestamp=1732020760684Num uniques 1053; 2024-11-19T12:52:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-19T12:52:40,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-19T12:52:40,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732020760707, max timestamp=1732020760718Num uniques 1012; 2024-11-19T12:52:40,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732020760707, max timestamp=1732020760718Num uniques 1012; 2024-11-19T12:52:40,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-19T12:52:40,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-19T12:52:40,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732020760707, max timestamp=1732020760718Num uniques 1012; 2024-11-19T12:52:40,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732020760730, max timestamp=1732020760741Num uniques 1071; 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732020760730, max timestamp=1732020760741Num uniques 1071; 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732020760730, max timestamp=1732020760741Num uniques 1071; 2024-11-19T12:52:40,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T12:52:40,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020820761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020820762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020820763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732020760760, max timestamp=1732020760765Num uniques 1130; 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T12:52:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732020760760, max timestamp=1732020760765Num uniques 1130; 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732020760760, max timestamp=1732020760765Num uniques 1130; 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T12:52:40,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T12:52:40,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732020760791, max timestamp=1732020760832Num uniques 1089; 2024-11-19T12:52:40,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732020760791, max timestamp=1732020760832Num uniques 1089; 2024-11-19T12:52:40,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-19T12:52:40,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-19T12:52:40,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732020760791, max timestamp=1732020760832Num uniques 1089; 2024-11-19T12:52:40,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-19T12:52:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020820873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1732020760847, max timestamp=1732020760875Num uniques 1048; 2024-11-19T12:52:40,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-11-19T12:52:40,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1732020760847, max timestamp=1732020760875Num uniques 1048; 2024-11-19T12:52:40,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-11-19T12:52:40,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1732020760847, max timestamp=1732020760875Num uniques 1048; 2024-11-19T12:52:40,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=11358, suffix heap size=136960,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=248 2024-11-19T12:52:40,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2bc5467872b04e2bb7d15f3f9e2a53d7 2024-11-19T12:52:40,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/62f13352c62d4511bb2d532042664bee is 50, key is test_row_0/C:col10/1732020759940/Put/seqid=0 2024-11-19T12:52:40,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742068_1244 (size=12001) 2024-11-19T12:52:40,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/62f13352c62d4511bb2d532042664bee 2024-11-19T12:52:40,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020820903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732020760900, max timestamp=1732020760909Num uniques 1107; 2024-11-19T12:52:40,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-19T12:52:40,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732020760900, max timestamp=1732020760909Num uniques 1107; 2024-11-19T12:52:40,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-19T12:52:40,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732020760900, max timestamp=1732020760909Num uniques 1107; 2024-11-19T12:52:40,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50695, new segment data size=9475, suffix heap size=143096,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1107, new segment cells count=207 2024-11-19T12:52:40,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/135b8955db7f4b9383962ddd503cb9b8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8 2024-11-19T12:52:40,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8, entries=250, sequenceid=82, filesize=16.3 K 2024-11-19T12:52:40,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2bc5467872b04e2bb7d15f3f9e2a53d7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7 2024-11-19T12:52:40,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7, entries=150, sequenceid=82, filesize=11.7 K 2024-11-19T12:52:40,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/62f13352c62d4511bb2d532042664bee as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee 2024-11-19T12:52:40,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee, entries=150, sequenceid=82, filesize=11.7 K 2024-11-19T12:52:40,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=107.34 KB/109920 for 8451cbb661c52001cbf3306ac11815b1 in 875ms, sequenceid=82, compaction requested=false 2024-11-19T12:52:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:40,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=114.05 KB heapSize=320.72 KB 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:40,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2341d736c5bf43b2b4dadd947410fcd7 is 50, key is test_row_0/A:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:40,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:40,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:40,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742069_1245 (size=14741) 2024-11-19T12:52:40,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2341d736c5bf43b2b4dadd947410fcd7 2024-11-19T12:52:40,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/452566529e1e4c92939b29a9308d275e is 50, key is test_row_0/B:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020820958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020820959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020820963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:40,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742070_1246 (size=12301) 2024-11-19T12:52:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020820981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020821021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020821071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020821071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020821071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020821191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020821232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020821281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020821281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020821281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/452566529e1e4c92939b29a9308d275e 2024-11-19T12:52:41,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/49e6493b5f0b47f8b28bd8187ae17640 is 50, key is test_row_0/C:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742071_1247 (size=12301) 2024-11-19T12:52:41,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/49e6493b5f0b47f8b28bd8187ae17640 2024-11-19T12:52:41,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2341d736c5bf43b2b4dadd947410fcd7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7 2024-11-19T12:52:41,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7, entries=200, sequenceid=496, filesize=14.4 K 2024-11-19T12:52:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/452566529e1e4c92939b29a9308d275e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e 2024-11-19T12:52:41,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-19T12:52:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/49e6493b5f0b47f8b28bd8187ae17640 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640 2024-11-19T12:52:41,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640, entries=150, sequenceid=496, filesize=12.0 K 2024-11-19T12:52:41,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.67 KB/328368, currentSize=73.80 KB/75570 for 8451cbb661c52001cbf3306ac11815b1 in 467ms, sequenceid=496, compaction requested=true 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:41,402 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:41,402 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:41,403 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:41,403 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:41,403 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,404 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/f330baadcde540f48c4e7de4c7d07e98, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=35.6 K 2024-11-19T12:52:41,404 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43526 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:41,404 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:41,404 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,404 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f330baadcde540f48c4e7de4c7d07e98, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:41,404 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/807b332108234c639fdcc75b00b89202, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=42.5 K 2024-11-19T12:52:41,405 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807b332108234c639fdcc75b00b89202, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:41,405 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc5467872b04e2bb7d15f3f9e2a53d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732020759940 2024-11-19T12:52:41,405 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135b8955db7f4b9383962ddd503cb9b8, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732020759930 2024-11-19T12:52:41,405 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 452566529e1e4c92939b29a9308d275e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760923 2024-11-19T12:52:41,406 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2341d736c5bf43b2b4dadd947410fcd7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760920 2024-11-19T12:52:41,424 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:41,425 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/4483cb6fd06a4e6891e0ad7f39786b30 is 50, key is test_row_0/B:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:41,425 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:41,426 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1edcf4d3ad8e48c7a97112da20653944 is 50, key is test_row_0/A:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:41,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742072_1248 (size=12507) 2024-11-19T12:52:41,453 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/4483cb6fd06a4e6891e0ad7f39786b30 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/4483cb6fd06a4e6891e0ad7f39786b30 2024-11-19T12:52:41,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742073_1249 (size=12507) 2024-11-19T12:52:41,460 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 4483cb6fd06a4e6891e0ad7f39786b30(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:41,460 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:41,460 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020761402; duration=0sec 2024-11-19T12:52:41,461 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:41,461 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:41,461 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:41,465 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:41,465 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1edcf4d3ad8e48c7a97112da20653944 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1edcf4d3ad8e48c7a97112da20653944 2024-11-19T12:52:41,465 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:41,466 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,466 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e75816481af14b7db7f4a7a609a227b3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=35.6 K 2024-11-19T12:52:41,466 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e75816481af14b7db7f4a7a609a227b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732020759270 2024-11-19T12:52:41,467 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f13352c62d4511bb2d532042664bee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732020759940 2024-11-19T12:52:41,467 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e6493b5f0b47f8b28bd8187ae17640, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760923 2024-11-19T12:52:41,470 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 1edcf4d3ad8e48c7a97112da20653944(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:41,470 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:41,470 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020761402; duration=0sec 2024-11-19T12:52:41,470 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:41,470 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:41,481 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#208 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:41,481 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/1b6ccb1832f7485eafe047d58fa72b2a is 50, key is test_row_0/C:col10/1732020760934/Put/seqid=0 2024-11-19T12:52:41,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742074_1250 (size=12507) 2024-11-19T12:52:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:41,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:41,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:41,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:41,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:41,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:41,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/38af8c2ac30d40a081472a82f005769a is 50, key is test_row_0/A:col10/1732020761503/Put/seqid=0 2024-11-19T12:52:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742075_1251 (size=14741) 2024-11-19T12:52:41,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/38af8c2ac30d40a081472a82f005769a 2024-11-19T12:52:41,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/1ea24113afe5450081d2e7ac795c44e1 is 50, key is test_row_0/B:col10/1732020761503/Put/seqid=0 2024-11-19T12:52:41,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742076_1252 (size=12301) 2024-11-19T12:52:41,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020821558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020821562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020821591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020821591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020821591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020821671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020821671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:41,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020821881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020821881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,886 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:52:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:41,906 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/1b6ccb1832f7485eafe047d58fa72b2a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/1b6ccb1832f7485eafe047d58fa72b2a 2024-11-19T12:52:41,911 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into 1b6ccb1832f7485eafe047d58fa72b2a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:41,911 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:41,911 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020761402; duration=0sec 2024-11-19T12:52:41,911 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:41,911 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:41,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/1ea24113afe5450081d2e7ac795c44e1 2024-11-19T12:52:41,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c11a2a98baf44e10bf795f60ba811d6a is 50, key is test_row_0/C:col10/1732020761503/Put/seqid=0 2024-11-19T12:52:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742077_1253 (size=12301) 2024-11-19T12:52:42,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020822102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020822102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020822102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020822191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020822191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:42,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c11a2a98baf44e10bf795f60ba811d6a 2024-11-19T12:52:42,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/38af8c2ac30d40a081472a82f005769a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a 2024-11-19T12:52:42,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a, entries=200, sequenceid=513, filesize=14.4 K 2024-11-19T12:52:42,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/1ea24113afe5450081d2e7ac795c44e1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1 2024-11-19T12:52:42,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1, entries=150, sequenceid=513, filesize=12.0 K 2024-11-19T12:52:42,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c11a2a98baf44e10bf795f60ba811d6a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a 2024-11-19T12:52:42,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a, entries=150, sequenceid=513, filesize=12.0 K 2024-11-19T12:52:42,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8451cbb661c52001cbf3306ac11815b1 in 863ms, sequenceid=513, compaction requested=false 2024-11-19T12:52:42,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:42,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,470 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a76c6efa507c467d90b3f6cf70161eb5 is 50, key is test_row_0/A:col10/1732020761549/Put/seqid=0 2024-11-19T12:52:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742078_1254 (size=12301) 2024-11-19T12:52:42,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020822723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020822726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020822831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020822841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:42,893 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a76c6efa507c467d90b3f6cf70161eb5 2024-11-19T12:52:42,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a05c604bf9bd414f9a84cf55bc55b956 is 50, key is test_row_0/B:col10/1732020761549/Put/seqid=0 2024-11-19T12:52:42,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742079_1255 (size=12301) 2024-11-19T12:52:42,906 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a05c604bf9bd414f9a84cf55bc55b956 2024-11-19T12:52:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/08c76c59c534456b820904b63eb1aa35 is 50, key is test_row_0/C:col10/1732020761549/Put/seqid=0 2024-11-19T12:52:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742080_1256 (size=12301) 2024-11-19T12:52:42,926 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/08c76c59c534456b820904b63eb1aa35 2024-11-19T12:52:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a76c6efa507c467d90b3f6cf70161eb5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5 2024-11-19T12:52:42,935 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5, entries=150, sequenceid=531, filesize=12.0 K 2024-11-19T12:52:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a05c604bf9bd414f9a84cf55bc55b956 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956 2024-11-19T12:52:42,939 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956, entries=150, sequenceid=531, filesize=12.0 K 2024-11-19T12:52:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/08c76c59c534456b820904b63eb1aa35 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35 2024-11-19T12:52:42,943 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35, entries=150, sequenceid=531, filesize=12.0 K 2024-11-19T12:52:42,944 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8451cbb661c52001cbf3306ac11815b1 in 474ms, sequenceid=531, compaction requested=true 2024-11-19T12:52:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:42,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T12:52:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T12:52:42,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T12:52:42,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9220 sec 2024-11-19T12:52:42,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.9260 sec 2024-11-19T12:52:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:43,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:43,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1b85dec835d94fefa77b941ad709d5a8 is 50, key is test_row_0/A:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742081_1257 (size=14741) 2024-11-19T12:52:43,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1b85dec835d94fefa77b941ad709d5a8 2024-11-19T12:52:43,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ff00fef2a4da49cfbbab8d000fa760b0 is 50, key is test_row_0/B:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742082_1258 (size=12301) 2024-11-19T12:52:43,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ff00fef2a4da49cfbbab8d000fa760b0 2024-11-19T12:52:43,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/da315eacff314327a8cdbec04a7d72a4 is 50, key is test_row_0/C:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020823081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020823084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742083_1259 (size=12301) 2024-11-19T12:52:43,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/da315eacff314327a8cdbec04a7d72a4 2024-11-19T12:52:43,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1b85dec835d94fefa77b941ad709d5a8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8 2024-11-19T12:52:43,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8, entries=200, sequenceid=547, filesize=14.4 K 2024-11-19T12:52:43,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ff00fef2a4da49cfbbab8d000fa760b0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0 2024-11-19T12:52:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0, entries=150, sequenceid=547, filesize=12.0 K 2024-11-19T12:52:43,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/da315eacff314327a8cdbec04a7d72a4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4 2024-11-19T12:52:43,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4, entries=150, sequenceid=547, filesize=12.0 K 2024-11-19T12:52:43,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020823111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,115 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8451cbb661c52001cbf3306ac11815b1 in 72ms, sequenceid=547, compaction requested=true 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:43,116 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:43,116 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:43,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:43,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:43,117 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:43,118 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1edcf4d3ad8e48c7a97112da20653944, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=53.0 K 2024-11-19T12:52:43,118 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:43,118 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:43,118 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:43,118 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/4483cb6fd06a4e6891e0ad7f39786b30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=48.3 K 2024-11-19T12:52:43,118 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1edcf4d3ad8e48c7a97112da20653944, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760923 2024-11-19T12:52:43,118 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4483cb6fd06a4e6891e0ad7f39786b30, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760923 2024-11-19T12:52:43,119 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea24113afe5450081d2e7ac795c44e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732020760943 2024-11-19T12:52:43,119 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38af8c2ac30d40a081472a82f005769a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732020760943 2024-11-19T12:52:43,120 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a05c604bf9bd414f9a84cf55bc55b956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732020761549 2024-11-19T12:52:43,120 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76c6efa507c467d90b3f6cf70161eb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732020761549 2024-11-19T12:52:43,121 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ff00fef2a4da49cfbbab8d000fa760b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762721 2024-11-19T12:52:43,121 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b85dec835d94fefa77b941ad709d5a8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762717 2024-11-19T12:52:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:43,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:52:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:43,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/3cdec09bb189446589bdf83e789d7c8b is 50, key is test_row_0/A:col10/1732020763079/Put/seqid=0 2024-11-19T12:52:43,132 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:43,132 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:43,133 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/938b7ba3c8464edda7403b6094a4c2ba is 50, key is test_row_0/A:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,133 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0ae530afeef842c29523b2163fedcb29 is 50, key is test_row_0/B:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742086_1262 (size=12643) 2024-11-19T12:52:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742085_1261 (size=12643) 2024-11-19T12:52:43,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742084_1260 (size=12301) 2024-11-19T12:52:43,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/3cdec09bb189446589bdf83e789d7c8b 2024-11-19T12:52:43,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/65eae8c85c0a44999cc0ae4bd701ef68 is 50, key is test_row_0/B:col10/1732020763079/Put/seqid=0 2024-11-19T12:52:43,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020823160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020823160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742087_1263 (size=12301) 2024-11-19T12:52:43,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020823191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020823202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020823271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020823271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020823401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020823411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020823481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020823481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,556 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/938b7ba3c8464edda7403b6094a4c2ba as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/938b7ba3c8464edda7403b6094a4c2ba 2024-11-19T12:52:43,560 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 938b7ba3c8464edda7403b6094a4c2ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:43,560 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:43,560 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=12, startTime=1732020763116; duration=0sec 2024-11-19T12:52:43,561 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:43,561 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:43,561 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:43,562 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:43,562 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:43,562 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:43,562 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/1b6ccb1832f7485eafe047d58fa72b2a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=48.3 K 2024-11-19T12:52:43,562 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0ae530afeef842c29523b2163fedcb29 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0ae530afeef842c29523b2163fedcb29 2024-11-19T12:52:43,562 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6ccb1832f7485eafe047d58fa72b2a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732020760923 2024-11-19T12:52:43,563 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11a2a98baf44e10bf795f60ba811d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732020760943 2024-11-19T12:52:43,563 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08c76c59c534456b820904b63eb1aa35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732020761549 2024-11-19T12:52:43,563 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting da315eacff314327a8cdbec04a7d72a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762721 2024-11-19T12:52:43,566 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 0ae530afeef842c29523b2163fedcb29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:43,566 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:43,566 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=12, startTime=1732020763116; duration=0sec 2024-11-19T12:52:43,566 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:43,566 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:43,570 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:43,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/65eae8c85c0a44999cc0ae4bd701ef68 2024-11-19T12:52:43,571 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ff3b47fdad414dc8baf811738026ca6b is 50, key is test_row_0/C:col10/1732020763043/Put/seqid=0 2024-11-19T12:52:43,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742088_1264 (size=12643) 2024-11-19T12:52:43,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/acdf051309344008b377b2826944cc36 is 50, key is test_row_0/C:col10/1732020763079/Put/seqid=0 2024-11-19T12:52:43,581 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ff3b47fdad414dc8baf811738026ca6b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ff3b47fdad414dc8baf811738026ca6b 2024-11-19T12:52:43,585 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into ff3b47fdad414dc8baf811738026ca6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:43,585 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:43,585 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=12, startTime=1732020763116; duration=0sec 2024-11-19T12:52:43,585 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:43,585 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:43,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742089_1265 (size=12301) 2024-11-19T12:52:43,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020823711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020823721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020823791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020823791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:43,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/acdf051309344008b377b2826944cc36 2024-11-19T12:52:43,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/3cdec09bb189446589bdf83e789d7c8b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b 2024-11-19T12:52:43,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b, entries=150, sequenceid=564, filesize=12.0 K 2024-11-19T12:52:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/65eae8c85c0a44999cc0ae4bd701ef68 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68 2024-11-19T12:52:44,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68, entries=150, sequenceid=564, filesize=12.0 K 2024-11-19T12:52:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/acdf051309344008b377b2826944cc36 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36 2024-11-19T12:52:44,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36, entries=150, sequenceid=564, filesize=12.0 K 2024-11-19T12:52:44,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 8451cbb661c52001cbf3306ac11815b1 in 885ms, sequenceid=564, compaction requested=false 2024-11-19T12:52:44,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T12:52:44,181 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:44,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T12:52:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:44,183 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:44,184 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:44,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:44,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T12:52:44,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:44,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:44,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:44,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2dd65ce8b3504dd1af9c5359786e7c92 is 50, key is test_row_0/A:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:44,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742090_1266 (size=12301) 2024-11-19T12:52:44,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2dd65ce8b3504dd1af9c5359786e7c92 2024-11-19T12:52:44,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8666afc248a44bf8839d927dde302359 is 50, key is test_row_0/B:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:44,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742091_1267 (size=12301) 2024-11-19T12:52:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020824266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020824269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020824301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020824301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020824371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020824381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:44,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020824581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020824591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8666afc248a44bf8839d927dde302359 2024-11-19T12:52:44,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c7f5d04bc4904cc59eb395dfd4361ba2 is 50, key is test_row_0/C:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742092_1268 (size=12301) 2024-11-19T12:52:44,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020824891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020824901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:44,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:44,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:45,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c7f5d04bc4904cc59eb395dfd4361ba2 2024-11-19T12:52:45,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2dd65ce8b3504dd1af9c5359786e7c92 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92 2024-11-19T12:52:45,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92, entries=150, sequenceid=584, filesize=12.0 K 2024-11-19T12:52:45,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8666afc248a44bf8839d927dde302359 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359 2024-11-19T12:52:45,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359, entries=150, sequenceid=584, filesize=12.0 K 2024-11-19T12:52:45,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/c7f5d04bc4904cc59eb395dfd4361ba2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2 2024-11-19T12:52:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2, entries=150, sequenceid=584, filesize=12.0 K 2024-11-19T12:52:45,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8451cbb661c52001cbf3306ac11815b1 in 866ms, sequenceid=584, compaction requested=true 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:45,089 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:45,089 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:45,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:45,090 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:45,090 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:45,090 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/938b7ba3c8464edda7403b6094a4c2ba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.4 K 2024-11-19T12:52:45,090 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0ae530afeef842c29523b2163fedcb29, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.4 K 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938b7ba3c8464edda7403b6094a4c2ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762721 2024-11-19T12:52:45,090 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae530afeef842c29523b2163fedcb29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762721 2024-11-19T12:52:45,091 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 65eae8c85c0a44999cc0ae4bd701ef68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732020763061 2024-11-19T12:52:45,091 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cdec09bb189446589bdf83e789d7c8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732020763061 2024-11-19T12:52:45,091 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8666afc248a44bf8839d927dde302359, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:45,091 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd65ce8b3504dd1af9c5359786e7c92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:45,098 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:45,099 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a0cb170cccbb4682a08e7f921b1e71e4 is 50, key is test_row_0/A:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T12:52:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:45,100 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:45,101 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/001f342b6c004fc5b195673930a2fe70 is 50, key is test_row_0/B:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:45,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0e674857b5a347268217bca426d8c578 is 50, key is test_row_0/A:col10/1732020764260/Put/seqid=0 2024-11-19T12:52:45,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742093_1269 (size=12745) 2024-11-19T12:52:45,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742094_1270 (size=12745) 2024-11-19T12:52:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:45,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742095_1271 (size=12301) 2024-11-19T12:52:45,143 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0e674857b5a347268217bca426d8c578 2024-11-19T12:52:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d1674553119d4f4f9de7f2019dfd4d10 is 50, key is test_row_0/B:col10/1732020764260/Put/seqid=0 2024-11-19T12:52:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742096_1272 (size=12301) 2024-11-19T12:52:45,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020825192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020825302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020825311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020825311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,314 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:45,314 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:45,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020825401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020825411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020825511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,531 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a0cb170cccbb4682a08e7f921b1e71e4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a0cb170cccbb4682a08e7f921b1e71e4 2024-11-19T12:52:45,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into a0cb170cccbb4682a08e7f921b1e71e4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:45,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:45,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020765089; duration=0sec 2024-11-19T12:52:45,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:45,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:45,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:45,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/001f342b6c004fc5b195673930a2fe70 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/001f342b6c004fc5b195673930a2fe70 2024-11-19T12:52:45,537 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:45,537 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:45,537 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:45,537 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ff3b47fdad414dc8baf811738026ca6b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.4 K 2024-11-19T12:52:45,537 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3b47fdad414dc8baf811738026ca6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1732020762721 2024-11-19T12:52:45,537 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting acdf051309344008b377b2826944cc36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732020763061 2024-11-19T12:52:45,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7f5d04bc4904cc59eb395dfd4361ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:45,540 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 001f342b6c004fc5b195673930a2fe70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:45,540 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:45,540 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020765089; duration=0sec 2024-11-19T12:52:45,540 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:45,540 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:45,545 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:45,546 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/98a4e467ed584c85b9adf746a8f22d9a is 50, key is test_row_0/C:col10/1732020763159/Put/seqid=0 2024-11-19T12:52:45,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742097_1273 (size=12745) 2024-11-19T12:52:45,556 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d1674553119d4f4f9de7f2019dfd4d10 2024-11-19T12:52:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/f727094b80bc41d09dbcc9fd19979c80 is 50, key is test_row_0/C:col10/1732020764260/Put/seqid=0 2024-11-19T12:52:45,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742098_1274 (size=12301) 2024-11-19T12:52:45,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020825821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:45,955 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/98a4e467ed584c85b9adf746a8f22d9a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/98a4e467ed584c85b9adf746a8f22d9a 2024-11-19T12:52:45,959 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into 98a4e467ed584c85b9adf746a8f22d9a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:45,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:45,959 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020765089; duration=0sec 2024-11-19T12:52:45,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:45,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:45,967 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/f727094b80bc41d09dbcc9fd19979c80 2024-11-19T12:52:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0e674857b5a347268217bca426d8c578 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578 2024-11-19T12:52:45,975 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578, entries=150, sequenceid=599, filesize=12.0 K 2024-11-19T12:52:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d1674553119d4f4f9de7f2019dfd4d10 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10 2024-11-19T12:52:45,979 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10, entries=150, sequenceid=599, filesize=12.0 K 2024-11-19T12:52:45,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/f727094b80bc41d09dbcc9fd19979c80 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80 2024-11-19T12:52:45,984 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80, entries=150, sequenceid=599, filesize=12.0 K 2024-11-19T12:52:45,985 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8451cbb661c52001cbf3306ac11815b1 in 885ms, sequenceid=599, compaction requested=false 2024-11-19T12:52:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T12:52:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T12:52:45,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T12:52:45,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-19T12:52:45,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8060 sec 2024-11-19T12:52:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T12:52:46,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:46,331 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T12:52:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:46,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:46,379 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:46,380 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:46,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:46,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/996b71ec1f604d5fb6b224d56067f6e7 is 50, key is test_row_0/A:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:46,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742099_1275 (size=14741) 2024-11-19T12:52:46,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/996b71ec1f604d5fb6b224d56067f6e7 2024-11-19T12:52:46,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2d61ba44db374350b95abe6500c9fdcf is 50, key is test_row_0/B:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:46,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742100_1276 (size=12301) 2024-11-19T12:52:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020826424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020826425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020826425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:46,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:46,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:46,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020826531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020826531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020826531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020826741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020826741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020826741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:46,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:46,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2d61ba44db374350b95abe6500c9fdcf 2024-11-19T12:52:46,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/fb34c79a92c8437bb42ad4d5a435a8d0 is 50, key is test_row_0/C:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:46,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742101_1277 (size=12301) 2024-11-19T12:52:46,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:46,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:46,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:47,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020827051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020827051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020827051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:47,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:47,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/fb34c79a92c8437bb42ad4d5a435a8d0 2024-11-19T12:52:47,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/996b71ec1f604d5fb6b224d56067f6e7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7 2024-11-19T12:52:47,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7, entries=200, sequenceid=620, filesize=14.4 K 2024-11-19T12:52:47,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2d61ba44db374350b95abe6500c9fdcf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf 2024-11-19T12:52:47,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf, entries=150, sequenceid=620, filesize=12.0 K 2024-11-19T12:52:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/fb34c79a92c8437bb42ad4d5a435a8d0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0 2024-11-19T12:52:47,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0, entries=150, sequenceid=620, filesize=12.0 K 2024-11-19T12:52:47,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 8451cbb661c52001cbf3306ac11815b1 in 848ms, sequenceid=620, compaction requested=true 2024-11-19T12:52:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:47,228 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:47,229 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,229 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,229 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/001f342b6c004fc5b195673930a2fe70, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.5 K 2024-11-19T12:52:47,229 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a0cb170cccbb4682a08e7f921b1e71e4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=38.9 K 2024-11-19T12:52:47,229 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 001f342b6c004fc5b195673930a2fe70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:47,229 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0cb170cccbb4682a08e7f921b1e71e4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:47,230 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e674857b5a347268217bca426d8c578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732020764224 2024-11-19T12:52:47,230 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d1674553119d4f4f9de7f2019dfd4d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732020764224 2024-11-19T12:52:47,230 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996b71ec1f604d5fb6b224d56067f6e7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765175 2024-11-19T12:52:47,230 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d61ba44db374350b95abe6500c9fdcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765184 2024-11-19T12:52:47,235 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:47,236 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:47,236 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/3c218ea1ff3b40658a308f6f6d2b8cba is 50, key is test_row_0/A:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:47,236 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/1a4d23946e5f4d938b3c4ba5f776f97a is 50, key is test_row_0/B:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:47,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742103_1279 (size=12847) 2024-11-19T12:52:47,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742102_1278 (size=12847) 2024-11-19T12:52:47,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T12:52:47,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,297 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1d857781ffe845c8bf379e6d6774b997 is 50, key is test_row_0/A:col10/1732020766419/Put/seqid=0 2024-11-19T12:52:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742104_1280 (size=12301) 2024-11-19T12:52:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:47,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020827386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020827386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020827501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020827501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020827561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020827562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020827562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,644 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/1a4d23946e5f4d938b3c4ba5f776f97a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1a4d23946e5f4d938b3c4ba5f776f97a 2024-11-19T12:52:47,648 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 1a4d23946e5f4d938b3c4ba5f776f97a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:47,649 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:47,649 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020767228; duration=0sec 2024-11-19T12:52:47,649 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:47,649 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:47,649 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:47,650 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:47,650 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:47,650 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:47,650 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/98a4e467ed584c85b9adf746a8f22d9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.5 K 2024-11-19T12:52:47,651 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a4e467ed584c85b9adf746a8f22d9a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1732020763140 2024-11-19T12:52:47,651 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f727094b80bc41d09dbcc9fd19979c80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1732020764224 2024-11-19T12:52:47,652 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/3c218ea1ff3b40658a308f6f6d2b8cba as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3c218ea1ff3b40658a308f6f6d2b8cba 2024-11-19T12:52:47,652 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting fb34c79a92c8437bb42ad4d5a435a8d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765184 2024-11-19T12:52:47,657 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 3c218ea1ff3b40658a308f6f6d2b8cba(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:47,657 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:47,657 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020767227; duration=0sec 2024-11-19T12:52:47,657 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:47,657 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:47,657 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:47,658 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/0da7cc98f95b4bdda0feb8bf251a005f is 50, key is test_row_0/C:col10/1732020765191/Put/seqid=0 2024-11-19T12:52:47,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742105_1281 (size=12847) 2024-11-19T12:52:47,674 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/0da7cc98f95b4bdda0feb8bf251a005f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/0da7cc98f95b4bdda0feb8bf251a005f 2024-11-19T12:52:47,680 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into 0da7cc98f95b4bdda0feb8bf251a005f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:47,680 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:47,680 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020767228; duration=0sec 2024-11-19T12:52:47,680 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:47,680 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:47,705 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1d857781ffe845c8bf379e6d6774b997 2024-11-19T12:52:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020827711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020827712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/62d20695419f402f89bd82f4a6f7641d is 50, key is test_row_0/B:col10/1732020766419/Put/seqid=0 2024-11-19T12:52:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742106_1282 (size=12301) 2024-11-19T12:52:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020828021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020828021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,119 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/62d20695419f402f89bd82f4a6f7641d 2024-11-19T12:52:48,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/26c60c306e7344428619541680b44f17 is 50, key is test_row_0/C:col10/1732020766419/Put/seqid=0 2024-11-19T12:52:48,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742107_1283 (size=12301) 2024-11-19T12:52:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:48,530 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/26c60c306e7344428619541680b44f17 2024-11-19T12:52:48,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020828531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020828531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1d857781ffe845c8bf379e6d6774b997 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997 2024-11-19T12:52:48,538 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997, entries=150, sequenceid=635, filesize=12.0 K 2024-11-19T12:52:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/62d20695419f402f89bd82f4a6f7641d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d 2024-11-19T12:52:48,549 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d, entries=150, sequenceid=635, filesize=12.0 K 2024-11-19T12:52:48,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/26c60c306e7344428619541680b44f17 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17 2024-11-19T12:52:48,553 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17, entries=150, sequenceid=635, filesize=12.0 K 2024-11-19T12:52:48,554 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8451cbb661c52001cbf3306ac11815b1 in 1258ms, sequenceid=635, compaction requested=false 2024-11-19T12:52:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T12:52:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T12:52:48,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T12:52:48,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-19T12:52:48,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2260 sec 2024-11-19T12:52:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:48,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:48,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/bb08f7cb088c4cad8389d7b15fb17230 is 50, key is test_row_0/A:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:48,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742108_1284 (size=14737) 2024-11-19T12:52:48,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020828588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020828591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020828592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020828701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020828701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020828701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020828911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020828911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020828911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:48,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/bb08f7cb088c4cad8389d7b15fb17230 2024-11-19T12:52:48,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 is 50, key is test_row_0/B:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742109_1285 (size=9857) 2024-11-19T12:52:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020829221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020829222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020829222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 2024-11-19T12:52:49,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/74bbca0fd0db4bc099275331b9c6dea5 is 50, key is test_row_0/C:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:49,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742110_1286 (size=9857) 2024-11-19T12:52:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020829542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,547 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020829552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,556 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020829732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020829733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020829732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:49,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/74bbca0fd0db4bc099275331b9c6dea5 2024-11-19T12:52:49,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/bb08f7cb088c4cad8389d7b15fb17230 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230 2024-11-19T12:52:49,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230, entries=200, sequenceid=657, filesize=14.4 K 2024-11-19T12:52:49,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 2024-11-19T12:52:49,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5, entries=100, sequenceid=657, filesize=9.6 K 2024-11-19T12:52:49,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/74bbca0fd0db4bc099275331b9c6dea5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5 2024-11-19T12:52:49,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5, entries=100, sequenceid=657, filesize=9.6 K 2024-11-19T12:52:49,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8451cbb661c52001cbf3306ac11815b1 in 1318ms, sequenceid=657, compaction requested=true 2024-11-19T12:52:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:49,891 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:49,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:49,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:49,892 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:49,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:49,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:49,893 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39885 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:49,893 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:49,893 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:49,893 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3c218ea1ff3b40658a308f6f6d2b8cba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=39.0 K 2024-11-19T12:52:49,893 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:49,893 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c218ea1ff3b40658a308f6f6d2b8cba, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765184 2024-11-19T12:52:49,893 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:49,894 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:49,894 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1a4d23946e5f4d938b3c4ba5f776f97a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=34.2 K 2024-11-19T12:52:49,894 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d857781ffe845c8bf379e6d6774b997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732020766419 2024-11-19T12:52:49,894 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4d23946e5f4d938b3c4ba5f776f97a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765184 2024-11-19T12:52:49,894 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb08f7cb088c4cad8389d7b15fb17230, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020767363 2024-11-19T12:52:49,895 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d20695419f402f89bd82f4a6f7641d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732020766419 2024-11-19T12:52:49,895 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb7e083b68b4d3b9f4b7efe2eb30fc5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020768572 2024-11-19T12:52:49,955 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:49,955 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9b876b3e51a8412ebf18e2b93d2a6ca0 is 50, key is test_row_0/B:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:49,956 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:49,957 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2ed2551c59314fa0a4f588b356075959 is 50, key is test_row_0/A:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:49,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742112_1288 (size=12949) 2024-11-19T12:52:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742111_1287 (size=12949) 2024-11-19T12:52:50,005 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9b876b3e51a8412ebf18e2b93d2a6ca0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9b876b3e51a8412ebf18e2b93d2a6ca0 2024-11-19T12:52:50,010 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2ed2551c59314fa0a4f588b356075959 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2ed2551c59314fa0a4f588b356075959 2024-11-19T12:52:50,011 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 9b876b3e51a8412ebf18e2b93d2a6ca0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:50,011 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:50,011 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020769892; duration=0sec 2024-11-19T12:52:50,011 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:50,011 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:50,011 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:50,022 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:50,022 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:50,022 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:50,022 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/0da7cc98f95b4bdda0feb8bf251a005f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=34.2 K 2024-11-19T12:52:50,023 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da7cc98f95b4bdda0feb8bf251a005f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732020765184 2024-11-19T12:52:50,024 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c60c306e7344428619541680b44f17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732020766419 2024-11-19T12:52:50,024 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bbca0fd0db4bc099275331b9c6dea5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020768572 2024-11-19T12:52:50,026 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 2ed2551c59314fa0a4f588b356075959(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:50,026 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:50,026 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020769891; duration=0sec 2024-11-19T12:52:50,026 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:50,026 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:50,039 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:50,040 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/40d17e657fa64410b01fbbd9d86523c1 is 50, key is test_row_0/C:col10/1732020768572/Put/seqid=0 2024-11-19T12:52:50,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742113_1289 (size=12949) 2024-11-19T12:52:50,462 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/40d17e657fa64410b01fbbd9d86523c1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/40d17e657fa64410b01fbbd9d86523c1 2024-11-19T12:52:50,469 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into 40d17e657fa64410b01fbbd9d86523c1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:50,469 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:50,470 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020769892; duration=0sec 2024-11-19T12:52:50,470 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:50,470 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T12:52:50,532 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:50,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T12:52:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T12:52:50,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:50,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:50,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T12:52:50,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T12:52:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:50,690 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:52:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b932f9a9f8cc4a43985f80d498c15e95 is 50, key is test_row_0/A:col10/1732020768590/Put/seqid=0 2024-11-19T12:52:50,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742114_1290 (size=12301) 2024-11-19T12:52:50,750 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b932f9a9f8cc4a43985f80d498c15e95 2024-11-19T12:52:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:50,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d049e4c2585b4b41acf7274b41173ea6 is 50, key is test_row_0/B:col10/1732020768590/Put/seqid=0 2024-11-19T12:52:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020830795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:50,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020830803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020830803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:50,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742115_1291 (size=12301) 2024-11-19T12:52:50,819 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d049e4c2585b4b41acf7274b41173ea6 2024-11-19T12:52:50,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a8cc30f01f67472a9604f9213646d1fe is 50, key is test_row_0/C:col10/1732020768590/Put/seqid=0 2024-11-19T12:52:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T12:52:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742116_1292 (size=12301) 2024-11-19T12:52:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020830912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020830912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020830912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020831121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020831121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020831122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T12:52:51,269 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a8cc30f01f67472a9604f9213646d1fe 2024-11-19T12:52:51,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b932f9a9f8cc4a43985f80d498c15e95 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95 2024-11-19T12:52:51,290 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95, entries=150, sequenceid=673, filesize=12.0 K 2024-11-19T12:52:51,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/d049e4c2585b4b41acf7274b41173ea6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6 2024-11-19T12:52:51,296 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6, entries=150, sequenceid=673, filesize=12.0 K 2024-11-19T12:52:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a8cc30f01f67472a9604f9213646d1fe as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe 2024-11-19T12:52:51,301 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe, entries=150, sequenceid=673, filesize=12.0 K 2024-11-19T12:52:51,302 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8451cbb661c52001cbf3306ac11815b1 in 612ms, sequenceid=673, compaction requested=false 2024-11-19T12:52:51,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:51,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:51,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T12:52:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T12:52:51,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T12:52:51,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 768 msec 2024-11-19T12:52:51,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 774 msec 2024-11-19T12:52:51,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T12:52:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020770790, max timestamp=1732020771434Num uniques 859; 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020770790, max timestamp=1732020771434Num uniques 859; 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020770790, max timestamp=1732020771434Num uniques 859; 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:51,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:51,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0be2bbfd5a38440fb11f8d5fd4aab271 is 50, key is test_row_0/A:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:51,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742117_1293 (size=12301) 2024-11-19T12:52:51,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0be2bbfd5a38440fb11f8d5fd4aab271 2024-11-19T12:52:51,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/7859ec917b7a4edd8539fae65c145cb1 is 50, key is test_row_0/B:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020771467, max timestamp=1732020771483Num uniques 859; 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020771467, max timestamp=1732020771483Num uniques 859; 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020771467, max timestamp=1732020771483Num uniques 859; 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:52:51,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742118_1294 (size=12301) 2024-11-19T12:52:51,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020771505, max timestamp=1732020771550Num uniques 1018; 2024-11-19T12:52:51,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:52:51,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020771505, max timestamp=1732020771550Num uniques 1018; 2024-11-19T12:52:51,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:52:51,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020771505, max timestamp=1732020771550Num uniques 1018; 2024-11-19T12:52:51,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732020771573, max timestamp=1732020771581Num uniques 1077; 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732020771573, max timestamp=1732020771581Num uniques 1077; 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732020771573, max timestamp=1732020771581Num uniques 1077; 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-19T12:52:51,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-19T12:52:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020831604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020771601, max timestamp=1732020771609Num uniques 1136; 2024-11-19T12:52:51,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-19T12:52:51,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020771601, max timestamp=1732020771609Num uniques 1136; 2024-11-19T12:52:51,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-19T12:52:51,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020771601, max timestamp=1732020771609Num uniques 1136; 2024-11-19T12:52:51,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-19T12:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T12:52:51,672 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:51,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T12:52:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:51,674 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:51,674 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:51,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020831646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732020771638, max timestamp=1732020771663Num uniques 1095; 2024-11-19T12:52:51,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T12:52:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732020771638, max timestamp=1732020771663Num uniques 1095; 2024-11-19T12:52:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T12:52:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732020771638, max timestamp=1732020771663Num uniques 1095; 2024-11-19T12:52:51,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T12:52:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020831757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020831757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020831757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020771689, max timestamp=1732020771761Num uniques 1104; 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020771689, max timestamp=1732020771761Num uniques 1104; 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020771689, max timestamp=1732020771761Num uniques 1104; 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T12:52:51,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:51,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T12:52:51,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:51,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:51,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:51,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732020771801, max timestamp=1732020771825Num uniques 1013; 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732020771801, max timestamp=1732020771825Num uniques 1013; 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732020771801, max timestamp=1732020771825Num uniques 1013; 2024-11-19T12:52:51,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T12:52:51,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732020771857, max timestamp=1732020771871Num uniques 1072; 2024-11-19T12:52:51,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T12:52:51,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732020771857, max timestamp=1732020771871Num uniques 1072; 2024-11-19T12:52:51,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T12:52:51,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732020771857, max timestamp=1732020771871Num uniques 1072; 2024-11-19T12:52:51,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-19T12:52:51,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/7859ec917b7a4edd8539fae65c145cb1 2024-11-19T12:52:51,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/09247815ccec4ab99b450a806539064d is 50, key is test_row_0/C:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:51,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742119_1295 (size=12301) 2024-11-19T12:52:51,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020831942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020831943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10566, totalHeapSize=24264, min timestamp=1732020771903, max timestamp=1732020771953Num uniques 1081; 2024-11-19T12:52:51,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=10566, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T12:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10567, totalHeapSize=24264, min timestamp=1732020771903, max timestamp=1732020771953Num uniques 1031; 2024-11-19T12:52:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47207, new segment data size=10567, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-19T12:52:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10568, totalHeapSize=24264, min timestamp=1732020771903, max timestamp=1732020771953Num uniques 1081; 2024-11-19T12:52:51,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49498, new segment data size=10568, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T12:52:51,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T12:52:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:51,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:52,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1732020771993, max timestamp=1732020772010Num uniques 1090; 2024-11-19T12:52:52,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=8690, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-19T12:52:52,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1732020771993, max timestamp=1732020772010Num uniques 1090; 2024-11-19T12:52:52,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1732020771993, max timestamp=1732020772010Num uniques 1090; 2024-11-19T12:52:52,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=8690, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-19T12:52:52,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=8690, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-19T12:52:52,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732020772039, max timestamp=1732020772049Num uniques 1049; 2024-11-19T12:52:52,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-19T12:52:52,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732020772039, max timestamp=1732020772049Num uniques 1049; 2024-11-19T12:52:52,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-19T12:52:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732020772039, max timestamp=1732020772049Num uniques 1049; 2024-11-19T12:52:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-19T12:52:52,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14098, totalHeapSize=32272, min timestamp=1732020772067, max timestamp=1732020772096Num uniques 1108; 2024-11-19T12:52:52,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=14098, suffix heap size=143200,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=308 2024-11-19T12:52:52,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14098, totalHeapSize=32272, min timestamp=1732020772067, max timestamp=1732020772096Num uniques 1108; 2024-11-19T12:52:52,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=14098, suffix heap size=143200,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=308 2024-11-19T12:52:52,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1732020772067, max timestamp=1732020772096Num uniques 1108; 2024-11-19T12:52:52,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=14100, suffix heap size=143200,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=308 2024-11-19T12:52:52,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T12:52:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:52,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020832133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020832137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020832137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020832139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020832139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020832252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020832252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020832252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020832252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020832252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T12:52:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:52,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/09247815ccec4ab99b450a806539064d 2024-11-19T12:52:52,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0be2bbfd5a38440fb11f8d5fd4aab271 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271 2024-11-19T12:52:52,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T12:52:52,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/7859ec917b7a4edd8539fae65c145cb1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1 2024-11-19T12:52:52,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T12:52:52,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/09247815ccec4ab99b450a806539064d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d 2024-11-19T12:52:52,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T12:52:52,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 8451cbb661c52001cbf3306ac11815b1 in 891ms, sequenceid=695, compaction requested=true 2024-11-19T12:52:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:52,326 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:52,327 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:52,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:52,327 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:52,327 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,328 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:52,328 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2ed2551c59314fa0a4f588b356075959, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.7 K 2024-11-19T12:52:52,328 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:52,328 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,328 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9b876b3e51a8412ebf18e2b93d2a6ca0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.7 K 2024-11-19T12:52:52,329 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed2551c59314fa0a4f588b356075959, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020766424 2024-11-19T12:52:52,329 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b876b3e51a8412ebf18e2b93d2a6ca0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020766424 2024-11-19T12:52:52,330 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d049e4c2585b4b41acf7274b41173ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732020768584 2024-11-19T12:52:52,330 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b932f9a9f8cc4a43985f80d498c15e95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732020768584 2024-11-19T12:52:52,330 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be2bbfd5a38440fb11f8d5fd4aab271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:52,330 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7859ec917b7a4edd8539fae65c145cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:52,339 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:52,339 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ebdc85e1ebfc4626a665f3f103a0ac4e is 50, key is test_row_0/B:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:52,340 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#254 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:52,340 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/f62d0b1e01bf4cd8a4321f55ae9151f5 is 50, key is test_row_0/A:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:52,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742121_1297 (size=13051) 2024-11-19T12:52:52,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742120_1296 (size=13051) 2024-11-19T12:52:52,365 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ebdc85e1ebfc4626a665f3f103a0ac4e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ebdc85e1ebfc4626a665f3f103a0ac4e 2024-11-19T12:52:52,370 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into ebdc85e1ebfc4626a665f3f103a0ac4e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:52,370 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:52,370 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020772327; duration=0sec 2024-11-19T12:52:52,370 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:52,370 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:52,370 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:52,371 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:52,371 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:52,371 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,372 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/40d17e657fa64410b01fbbd9d86523c1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.7 K 2024-11-19T12:52:52,372 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d17e657fa64410b01fbbd9d86523c1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1732020766424 2024-11-19T12:52:52,373 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a8cc30f01f67472a9604f9213646d1fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732020768584 2024-11-19T12:52:52,374 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 09247815ccec4ab99b450a806539064d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:52,383 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:52,384 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/65eadf51bc444e79891a320ce4585dc9 is 50, key is test_row_0/C:col10/1732020770792/Put/seqid=0 2024-11-19T12:52:52,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742122_1298 (size=13051) 2024-11-19T12:52:52,411 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/65eadf51bc444e79891a320ce4585dc9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/65eadf51bc444e79891a320ce4585dc9 2024-11-19T12:52:52,420 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into 65eadf51bc444e79891a320ce4585dc9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:52,420 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:52,420 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020772327; duration=0sec 2024-11-19T12:52:52,420 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:52,420 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:52,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:52,439 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=154.31 KB heapSize=430.78 KB 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/370bea07dc4f47c8a2148b81dfdc817d is 50, key is test_row_0/A:col10/1732020772136/Put/seqid=0 2024-11-19T12:52:52,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742123_1299 (size=12301) 2024-11-19T12:52:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:52,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020832464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020832466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020832466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020832467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020832469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020832572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020832581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020832581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020832582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020832582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,752 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/f62d0b1e01bf4cd8a4321f55ae9151f5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f62d0b1e01bf4cd8a4321f55ae9151f5 2024-11-19T12:52:52,757 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into f62d0b1e01bf4cd8a4321f55ae9151f5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:52,757 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:52,757 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020772326; duration=0sec 2024-11-19T12:52:52,757 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:52,757 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020832781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020832791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020832791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020832791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020832791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:52,861 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=923 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/370bea07dc4f47c8a2148b81dfdc817d 2024-11-19T12:52:52,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0bb0448d217649baa0cf8644fbb2fa41 is 50, key is test_row_0/B:col10/1732020772136/Put/seqid=0 2024-11-19T12:52:52,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742124_1300 (size=12301) 2024-11-19T12:52:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020833091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020833101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020833101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020833102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020833102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,273 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=923 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0bb0448d217649baa0cf8644fbb2fa41 2024-11-19T12:52:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/b25028890f5a499989989e27fef1b2bf is 50, key is test_row_0/C:col10/1732020772136/Put/seqid=0 2024-11-19T12:52:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742125_1301 (size=12301) 2024-11-19T12:52:53,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020833601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020833611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020833611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020833612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020833612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:53,712 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=923 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/b25028890f5a499989989e27fef1b2bf 2024-11-19T12:52:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/370bea07dc4f47c8a2148b81dfdc817d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d 2024-11-19T12:52:53,722 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d, entries=150, sequenceid=923, filesize=12.0 K 2024-11-19T12:52:53,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0bb0448d217649baa0cf8644fbb2fa41 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41 2024-11-19T12:52:53,727 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41, entries=150, sequenceid=923, filesize=12.0 K 2024-11-19T12:52:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/b25028890f5a499989989e27fef1b2bf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf 2024-11-19T12:52:53,732 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf, entries=150, sequenceid=923, filesize=12.0 K 2024-11-19T12:52:53,733 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.73 KB/441072, currentSize=33.54 KB/34350 for 8451cbb661c52001cbf3306ac11815b1 in 1294ms, sequenceid=923, compaction requested=false 2024-11-19T12:52:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T12:52:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T12:52:53,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T12:52:53,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0610 sec 2024-11-19T12:52:53,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0650 sec 2024-11-19T12:52:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T12:52:53,821 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:53,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T12:52:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T12:52:53,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:53,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:53,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T12:52:53,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T12:52:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:53,976 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:53,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a4ef4862842a4ff8889c9a3f4ed6a82b is 50, key is test_row_0/A:col10/1732020772466/Put/seqid=0 2024-11-19T12:52:54,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742126_1302 (size=12301) 2024-11-19T12:52:54,011 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a4ef4862842a4ff8889c9a3f4ed6a82b 2024-11-19T12:52:54,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a7344792d43f4c57b375c6520894ad11 is 50, key is test_row_0/B:col10/1732020772466/Put/seqid=0 2024-11-19T12:52:54,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742127_1303 (size=12301) 2024-11-19T12:52:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T12:52:54,431 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a7344792d43f4c57b375c6520894ad11 2024-11-19T12:52:54,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ad191da173354255ab774af474b7b39a is 50, key is test_row_0/C:col10/1732020772466/Put/seqid=0 2024-11-19T12:52:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T12:52:54,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742128_1304 (size=12301) 2024-11-19T12:52:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:54,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732020774630, max timestamp=1732020774645Num uniques 859; 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732020774630, max timestamp=1732020774645Num uniques 859; 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732020774630, max timestamp=1732020774645Num uniques 859; 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:54,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:54,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732020774681, max timestamp=1732020774697Num uniques 1068; 2024-11-19T12:52:54,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T12:52:54,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732020774681, max timestamp=1732020774697Num uniques 1068; 2024-11-19T12:52:54,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732020774681, max timestamp=1732020774697Num uniques 1068; 2024-11-19T12:52:54,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T12:52:54,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-19T12:52:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020834716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020834721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020834721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020834723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020834723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020834832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020834832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020834832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020834832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020834832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:54,872 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ad191da173354255ab774af474b7b39a 2024-11-19T12:52:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a4ef4862842a4ff8889c9a3f4ed6a82b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b 2024-11-19T12:52:54,879 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b, entries=150, sequenceid=932, filesize=12.0 K 2024-11-19T12:52:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/a7344792d43f4c57b375c6520894ad11 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11 2024-11-19T12:52:54,884 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11, entries=150, sequenceid=932, filesize=12.0 K 2024-11-19T12:52:54,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ad191da173354255ab774af474b7b39a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a 2024-11-19T12:52:54,888 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a, entries=150, sequenceid=932, filesize=12.0 K 2024-11-19T12:52:54,891 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 8451cbb661c52001cbf3306ac11815b1 in 915ms, sequenceid=932, compaction requested=true 2024-11-19T12:52:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:54,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T12:52:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T12:52:54,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T12:52:54,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-19T12:52:54,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0740 sec 2024-11-19T12:52:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T12:52:54,961 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:54,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T12:52:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:54,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:54,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:54,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020774720, max timestamp=1732020775043Num uniques 1177; 2024-11-19T12:52:55,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-19T12:52:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020774720, max timestamp=1732020775043Num uniques 1177; 2024-11-19T12:52:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-19T12:52:55,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=129.71 KB heapSize=361.67 KB 2024-11-19T12:52:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020774720, max timestamp=1732020775043Num uniques 1177; 2024-11-19T12:52:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:55,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a28dd669d45d4ecf972e4eb69ea498ff is 50, key is test_row_0/A:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742129_1305 (size=19621) 2024-11-19T12:52:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732020775058, max timestamp=1732020775071Num uniques 859; 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020775058, max timestamp=1732020775071Num uniques 859; 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020775058, max timestamp=1732020775071Num uniques 859; 2024-11-19T12:52:55,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:55,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020835090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020835091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020835094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020835095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020835096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020835201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020835201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020835202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020835211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020835211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020835411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020835411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020835411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020835421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020835421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a28dd669d45d4ecf972e4eb69ea498ff 2024-11-19T12:52:55,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d28ece2bb6444bf9c5aee8319ea0e96 is 50, key is test_row_0/B:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:55,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742130_1306 (size=12301) 2024-11-19T12:52:55,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020835721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020835721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020835721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020835731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020835731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:55,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d28ece2bb6444bf9c5aee8319ea0e96 2024-11-19T12:52:55,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/e4432b2a955a44f68770651be013c290 is 50, key is test_row_0/C:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:55,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742131_1307 (size=12301) 2024-11-19T12:52:55,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/e4432b2a955a44f68770651be013c290 2024-11-19T12:52:55,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:55,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:55,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:55,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/a28dd669d45d4ecf972e4eb69ea498ff as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff 2024-11-19T12:52:55,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff, entries=300, sequenceid=990, filesize=19.2 K 2024-11-19T12:52:55,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/9d28ece2bb6444bf9c5aee8319ea0e96 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96 2024-11-19T12:52:55,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96, entries=150, sequenceid=990, filesize=12.0 K 2024-11-19T12:52:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/e4432b2a955a44f68770651be013c290 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290 2024-11-19T12:52:55,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290, entries=150, sequenceid=990, filesize=12.0 K 2024-11-19T12:52:55,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~124.27 KB/127248, currentSize=140.89 KB/144270 for 8451cbb661c52001cbf3306ac11815b1 in 854ms, sequenceid=990, compaction requested=true 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:55,901 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:55,901 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:55,903 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:55,903 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:55,903 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:55,903 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:55,903 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,903 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ebdc85e1ebfc4626a665f3f103a0ac4e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=48.8 K 2024-11-19T12:52:55,903 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:55,903 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f62d0b1e01bf4cd8a4321f55ae9151f5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=55.9 K 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdc85e1ebfc4626a665f3f103a0ac4e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62d0b1e01bf4cd8a4321f55ae9151f5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb0448d217649baa0cf8644fbb2fa41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=923, earliestPutTs=1732020772135 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370bea07dc4f47c8a2148b81dfdc817d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=923, earliestPutTs=1732020772135 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a7344792d43f4c57b375c6520894ad11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=932, earliestPutTs=1732020772464 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d28ece2bb6444bf9c5aee8319ea0e96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:55,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ef4862842a4ff8889c9a3f4ed6a82b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=932, earliestPutTs=1732020772464 2024-11-19T12:52:55,905 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28dd669d45d4ecf972e4eb69ea498ff, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:55,916 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:55,917 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1e263362c1bc43e5aca4ca3521f11c82 is 50, key is test_row_0/A:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:55,919 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:55,920 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ca6771da21594412b6599526df21e6bf is 50, key is test_row_0/B:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:55,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742132_1308 (size=13187) 2024-11-19T12:52:55,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742133_1309 (size=13187) 2024-11-19T12:52:56,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:56,033 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/f04a54d8aeda4edd81df11bff0e3ab83 is 50, key is test_row_0/A:col10/1732020775094/Put/seqid=0 2024-11-19T12:52:56,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742134_1310 (size=12301) 2024-11-19T12:52:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:56,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020836241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020836241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020836241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020836242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020836243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,328 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/1e263362c1bc43e5aca4ca3521f11c82 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1e263362c1bc43e5aca4ca3521f11c82 2024-11-19T12:52:56,333 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 1e263362c1bc43e5aca4ca3521f11c82(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:56,333 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:56,333 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=12, startTime=1732020775901; duration=0sec 2024-11-19T12:52:56,334 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:56,334 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:56,334 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:52:56,335 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:52:56,335 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:56,335 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:56,335 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/65eadf51bc444e79891a320ce4585dc9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=48.8 K 2024-11-19T12:52:56,336 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65eadf51bc444e79891a320ce4585dc9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732020770792 2024-11-19T12:52:56,336 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25028890f5a499989989e27fef1b2bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=923, earliestPutTs=1732020772135 2024-11-19T12:52:56,336 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad191da173354255ab774af474b7b39a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=932, earliestPutTs=1732020772464 2024-11-19T12:52:56,336 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4432b2a955a44f68770651be013c290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:56,342 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/ca6771da21594412b6599526df21e6bf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ca6771da21594412b6599526df21e6bf 2024-11-19T12:52:56,345 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:56,346 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a058252145654c9a9f960d7c841ece2b is 50, key is test_row_0/C:col10/1732020775046/Put/seqid=0 2024-11-19T12:52:56,346 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into ca6771da21594412b6599526df21e6bf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:56,346 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:56,346 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=12, startTime=1732020775901; duration=0sec 2024-11-19T12:52:56,347 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:56,347 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020836351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020836351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020836351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742135_1311 (size=13187) 2024-11-19T12:52:56,443 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1027 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/f04a54d8aeda4edd81df11bff0e3ab83 2024-11-19T12:52:56,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/649a8889a51b45e38995157e2dde8635 is 50, key is test_row_0/B:col10/1732020775094/Put/seqid=0 2024-11-19T12:52:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742136_1312 (size=12301) 2024-11-19T12:52:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020836561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020836561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020836561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,768 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a058252145654c9a9f960d7c841ece2b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a058252145654c9a9f960d7c841ece2b 2024-11-19T12:52:56,772 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into a058252145654c9a9f960d7c841ece2b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:56,772 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:56,772 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=12, startTime=1732020775901; duration=0sec 2024-11-19T12:52:56,772 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:56,772 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:56,857 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1027 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/649a8889a51b45e38995157e2dde8635 2024-11-19T12:52:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ab0dc2c699004dcfb9bce566a68f77df is 50, key is test_row_0/C:col10/1732020775094/Put/seqid=0 2024-11-19T12:52:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020836875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742137_1313 (size=12301) 2024-11-19T12:52:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020836875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020836875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:56,879 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1027 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ab0dc2c699004dcfb9bce566a68f77df 2024-11-19T12:52:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:56,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/f04a54d8aeda4edd81df11bff0e3ab83 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83 2024-11-19T12:52:56,889 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83, entries=150, sequenceid=1027, filesize=12.0 K 2024-11-19T12:52:56,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/649a8889a51b45e38995157e2dde8635 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635 2024-11-19T12:52:56,903 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635, entries=150, sequenceid=1027, filesize=12.0 K 2024-11-19T12:52:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ab0dc2c699004dcfb9bce566a68f77df as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df 2024-11-19T12:52:56,911 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df, entries=150, sequenceid=1027, filesize=12.0 K 2024-11-19T12:52:56,913 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8451cbb661c52001cbf3306ac11815b1 in 880ms, sequenceid=1027, compaction requested=false 2024-11-19T12:52:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T12:52:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T12:52:56,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T12:52:56,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9510 sec 2024-11-19T12:52:56,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9560 sec 2024-11-19T12:52:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T12:52:57,111 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:57,112 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T12:52:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T12:52:57,113 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:57,114 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:57,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T12:52:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:57,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:57,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T12:52:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/8e9a6347ad874bb19322ac98898a7eb6 is 50, key is test_row_0/A:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742138_1314 (size=12301) 2024-11-19T12:52:57,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1041 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/8e9a6347ad874bb19322ac98898a7eb6 2024-11-19T12:52:57,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0b07fb447f474137a9e3e73717cdd050 is 50, key is test_row_0/B:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742139_1315 (size=12301) 2024-11-19T12:52:57,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1041 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0b07fb447f474137a9e3e73717cdd050 2024-11-19T12:52:57,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d567a8b9a2334e52af594dd7fe4b8f19 is 50, key is test_row_0/C:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742140_1316 (size=12301) 2024-11-19T12:52:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020837305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020777294, max timestamp=1732020777307Num uniques 859; 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020777294, max timestamp=1732020777307Num uniques 859; 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020777294, max timestamp=1732020777307Num uniques 859; 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:57,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:52:57,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020837381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020837381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020837386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020837391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020837411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T12:52:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T12:52:57,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020837501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T12:52:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020837621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1041 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d567a8b9a2334e52af594dd7fe4b8f19 2024-11-19T12:52:57,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/8e9a6347ad874bb19322ac98898a7eb6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6 2024-11-19T12:52:57,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6, entries=150, sequenceid=1041, filesize=12.0 K 2024-11-19T12:52:57,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/0b07fb447f474137a9e3e73717cdd050 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050 2024-11-19T12:52:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020837711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050, entries=150, sequenceid=1041, filesize=12.0 K 2024-11-19T12:52:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d567a8b9a2334e52af594dd7fe4b8f19 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19 2024-11-19T12:52:57,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T12:52:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:57,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19, entries=150, sequenceid=1041, filesize=12.0 K 2024-11-19T12:52:57,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8451cbb661c52001cbf3306ac11815b1 in 475ms, sequenceid=1041, compaction requested=true 2024-11-19T12:52:57,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:57,738 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:57,739 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:57,740 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:57,740 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:57,740 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,741 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1e263362c1bc43e5aca4ca3521f11c82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.9 K 2024-11-19T12:52:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T12:52:57,741 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:57,741 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:57,741 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,741 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ca6771da21594412b6599526df21e6bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.9 K 2024-11-19T12:52:57,741 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e263362c1bc43e5aca4ca3521f11c82, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:57,742 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ca6771da21594412b6599526df21e6bf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:57,742 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04a54d8aeda4edd81df11bff0e3ab83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1027, earliestPutTs=1732020775088 2024-11-19T12:52:57,742 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 649a8889a51b45e38995157e2dde8635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1027, earliestPutTs=1732020775088 2024-11-19T12:52:57,743 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9a6347ad874bb19322ac98898a7eb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:57,743 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b07fb447f474137a9e3e73717cdd050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:57,765 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:57,766 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/e1658bf114144a44b22358da1ca600d5 is 50, key is test_row_0/A:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,785 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:57,785 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/67cc74f4e7884bf8b5dcbf40f82a282f is 50, key is test_row_0/B:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742141_1317 (size=13289) 2024-11-19T12:52:57,829 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/e1658bf114144a44b22358da1ca600d5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/e1658bf114144a44b22358da1ca600d5 2024-11-19T12:52:57,837 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into e1658bf114144a44b22358da1ca600d5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:57,837 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:57,837 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020777738; duration=0sec 2024-11-19T12:52:57,837 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:57,837 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:57,837 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:57,839 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:57,840 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:57,840 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,840 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a058252145654c9a9f960d7c841ece2b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=36.9 K 2024-11-19T12:52:57,841 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a058252145654c9a9f960d7c841ece2b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1732020774720 2024-11-19T12:52:57,841 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0dc2c699004dcfb9bce566a68f77df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1027, earliestPutTs=1732020775088 2024-11-19T12:52:57,842 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d567a8b9a2334e52af594dd7fe4b8f19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:57,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742142_1318 (size=13289) 2024-11-19T12:52:57,851 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:57,852 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ead8ca294f7b48949d75b6235e40f1fe is 50, key is test_row_0/C:col10/1732020776240/Put/seqid=0 2024-11-19T12:52:57,855 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/67cc74f4e7884bf8b5dcbf40f82a282f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/67cc74f4e7884bf8b5dcbf40f82a282f 2024-11-19T12:52:57,863 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 67cc74f4e7884bf8b5dcbf40f82a282f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:57,863 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:57,863 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020777739; duration=0sec 2024-11-19T12:52:57,863 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:57,863 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:57,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T12:52:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:57,882 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:52:57,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742143_1319 (size=13289) 2024-11-19T12:52:57,913 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ead8ca294f7b48949d75b6235e40f1fe as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ead8ca294f7b48949d75b6235e40f1fe 2024-11-19T12:52:57,920 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into ead8ca294f7b48949d75b6235e40f1fe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:57,920 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:57,920 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020777739; duration=0sec 2024-11-19T12:52:57,921 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:57,921 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0d7259dc5d6543e8a00ebac842f679dd is 50, key is test_row_0/A:col10/1732020777332/Put/seqid=0 2024-11-19T12:52:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:57,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742144_1320 (size=12301) 2024-11-19T12:52:57,983 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1078 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0d7259dc5d6543e8a00ebac842f679dd 2024-11-19T12:52:57,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020837983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:57,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8d3f978455e24415bfabc01e23592cd5 is 50, key is test_row_0/B:col10/1732020777332/Put/seqid=0 2024-11-19T12:52:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020838021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742145_1321 (size=12301) 2024-11-19T12:52:58,034 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1078 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8d3f978455e24415bfabc01e23592cd5 2024-11-19T12:52:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/648ab6866aae46cb85bd68535b618136 is 50, key is test_row_0/C:col10/1732020777332/Put/seqid=0 2024-11-19T12:52:58,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742146_1322 (size=12301) 2024-11-19T12:52:58,064 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1078 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/648ab6866aae46cb85bd68535b618136 2024-11-19T12:52:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/0d7259dc5d6543e8a00ebac842f679dd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd 2024-11-19T12:52:58,074 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd, entries=150, sequenceid=1078, filesize=12.0 K 2024-11-19T12:52:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8d3f978455e24415bfabc01e23592cd5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5 2024-11-19T12:52:58,084 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5, entries=150, sequenceid=1078, filesize=12.0 K 2024-11-19T12:52:58,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/648ab6866aae46cb85bd68535b618136 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136 2024-11-19T12:52:58,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020838091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,095 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136, entries=150, sequenceid=1078, filesize=12.0 K 2024-11-19T12:52:58,096 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 8451cbb661c52001cbf3306ac11815b1 in 214ms, sequenceid=1078, compaction requested=false 2024-11-19T12:52:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T12:52:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T12:52:58,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T12:52:58,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 986 msec 2024-11-19T12:52:58,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 991 msec 2024-11-19T12:52:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T12:52:58,251 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:52:58,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:52:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T12:52:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:52:58,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:52:58,255 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:52:58,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:52:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:58,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2976385553d94703805c5dd4baa30e59 is 50, key is test_row_0/A:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742147_1323 (size=9857) 2024-11-19T12:52:58,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1091 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2976385553d94703805c5dd4baa30e59 2024-11-19T12:52:58,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/91862f998ccf48c08a81c706227d635c is 50, key is test_row_0/B:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742148_1324 (size=9857) 2024-11-19T12:52:58,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1091 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/91862f998ccf48c08a81c706227d635c 2024-11-19T12:52:58,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/54904abecf064395bb02991bce627b47 is 50, key is test_row_0/C:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742149_1325 (size=9857) 2024-11-19T12:52:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:52:58,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020838379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38326 deadline: 1732020838402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38342 deadline: 1732020838402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,405 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,406 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T12:52:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38312 deadline: 1732020838411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,415 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 8451cbb661c52001cbf3306ac11815b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:52:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020838501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1732020838542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T12:52:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:52:58,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:52:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38344 deadline: 1732020838711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:52:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:52:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:52:58,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T12:52:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:52:58,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1091 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/54904abecf064395bb02991bce627b47 2024-11-19T12:52:58,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/2976385553d94703805c5dd4baa30e59 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59 2024-11-19T12:52:58,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59, entries=100, sequenceid=1091, filesize=9.6 K 2024-11-19T12:52:58,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/91862f998ccf48c08a81c706227d635c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c 2024-11-19T12:52:58,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c, entries=100, sequenceid=1091, filesize=9.6 K 2024-11-19T12:52:58,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/54904abecf064395bb02991bce627b47 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47 2024-11-19T12:52:58,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47, entries=100, sequenceid=1091, filesize=9.6 K 2024-11-19T12:52:58,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8451cbb661c52001cbf3306ac11815b1 in 459ms, sequenceid=1091, compaction requested=true 2024-11-19T12:52:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8451cbb661c52001cbf3306ac11815b1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:52:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/A is initiating minor compaction (all files) 2024-11-19T12:52:58,765 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/B is initiating minor compaction (all files) 2024-11-19T12:52:58,765 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/B in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,765 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/A in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,766 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/e1658bf114144a44b22358da1ca600d5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=34.6 K 2024-11-19T12:52:58,766 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/67cc74f4e7884bf8b5dcbf40f82a282f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=34.6 K 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 67cc74f4e7884bf8b5dcbf40f82a282f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1658bf114144a44b22358da1ca600d5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7259dc5d6543e8a00ebac842f679dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1078, earliestPutTs=1732020777332 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3f978455e24415bfabc01e23592cd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1078, earliestPutTs=1732020777332 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 91862f998ccf48c08a81c706227d635c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1091, earliestPutTs=1732020777979 2024-11-19T12:52:58,766 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2976385553d94703805c5dd4baa30e59, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1091, earliestPutTs=1732020777979 2024-11-19T12:52:58,772 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:58,772 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:58,772 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/47989bbd230d4b1494b8850cb96c117d is 50, key is test_row_0/A:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8a685562b67d4b40a40cd1fe026210f5 is 50, key is test_row_0/B:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742151_1327 (size=13391) 2024-11-19T12:52:58,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742150_1326 (size=13391) 2024-11-19T12:52:58,779 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/8a685562b67d4b40a40cd1fe026210f5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8a685562b67d4b40a40cd1fe026210f5 2024-11-19T12:52:58,783 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/B of 8451cbb661c52001cbf3306ac11815b1 into 8a685562b67d4b40a40cd1fe026210f5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:58,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:58,783 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/B, priority=13, startTime=1732020778765; duration=0sec 2024-11-19T12:52:58,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:52:58,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:B 2024-11-19T12:52:58,783 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:52:58,785 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:52:58,785 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 8451cbb661c52001cbf3306ac11815b1/C is initiating minor compaction (all files) 2024-11-19T12:52:58,785 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8451cbb661c52001cbf3306ac11815b1/C in TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,786 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ead8ca294f7b48949d75b6235e40f1fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp, totalSize=34.6 K 2024-11-19T12:52:58,786 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ead8ca294f7b48949d75b6235e40f1fe, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1041, earliestPutTs=1732020776240 2024-11-19T12:52:58,786 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 648ab6866aae46cb85bd68535b618136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1078, earliestPutTs=1732020777332 2024-11-19T12:52:58,786 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 54904abecf064395bb02991bce627b47, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=1091, earliestPutTs=1732020777979 2024-11-19T12:52:58,795 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8451cbb661c52001cbf3306ac11815b1#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:52:58,795 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a6bac24037794888bb892e28e30b7699 is 50, key is test_row_0/C:col10/1732020777979/Put/seqid=0 2024-11-19T12:52:58,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742152_1328 (size=13391) 2024-11-19T12:52:58,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T12:52:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:58,867 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:52:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:52:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:52:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:52:58,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:52:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/347d0bbf62cc4d6cbfa58007fa5e8c09 is 50, key is test_row_0/A:col10/1732020778369/Put/seqid=0 2024-11-19T12:52:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:52:58,893 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-19T12:52:58,893 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:58,893 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,893 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-19T12:52:58,893 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:58,894 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,894 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,894 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:58,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:58,894 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-19T12:52:58,894 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:58,894 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,894 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:58,896 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-19T12:52:58,896 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:58,896 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,896 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:58,897 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-19T12:52:58,897 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:58,897 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,898 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:58,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:58,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742153_1329 (size=12301) 2024-11-19T12:52:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:52:59,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. as already flushing 2024-11-19T12:52:59,023 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-19T12:52:59,023 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:59,023 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:59,024 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:59,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:59,190 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/47989bbd230d4b1494b8850cb96c117d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/47989bbd230d4b1494b8850cb96c117d 2024-11-19T12:52:59,204 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/a6bac24037794888bb892e28e30b7699 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a6bac24037794888bb892e28e30b7699 2024-11-19T12:52:59,204 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/A of 8451cbb661c52001cbf3306ac11815b1 into 47989bbd230d4b1494b8850cb96c117d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:59,204 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:59,204 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/A, priority=13, startTime=1732020778765; duration=0sec 2024-11-19T12:52:59,204 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:59,204 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:A 2024-11-19T12:52:59,209 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8451cbb661c52001cbf3306ac11815b1/C of 8451cbb661c52001cbf3306ac11815b1 into a6bac24037794888bb892e28e30b7699(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:52:59,209 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:59,209 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., storeName=8451cbb661c52001cbf3306ac11815b1/C, priority=13, startTime=1732020778765; duration=0sec 2024-11-19T12:52:59,209 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:52:59,209 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8451cbb661c52001cbf3306ac11815b1:C 2024-11-19T12:52:59,301 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1112 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/347d0bbf62cc4d6cbfa58007fa5e8c09 2024-11-19T12:52:59,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/aa19d1625b104a7592d02b35d80fc948 is 50, key is test_row_0/B:col10/1732020778369/Put/seqid=0 2024-11-19T12:52:59,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742154_1330 (size=12301) 2024-11-19T12:52:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:52:59,563 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-19T12:52:59,563 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:52:59,563 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:59,563 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:52:59,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:52:59,713 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1112 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/aa19d1625b104a7592d02b35d80fc948 2024-11-19T12:52:59,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ef3241226d38461895f4ba6c906ccde8 is 50, key is test_row_0/C:col10/1732020778369/Put/seqid=0 2024-11-19T12:52:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742155_1331 (size=12301) 2024-11-19T12:52:59,724 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1112 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ef3241226d38461895f4ba6c906ccde8 2024-11-19T12:52:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/347d0bbf62cc4d6cbfa58007fa5e8c09 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/347d0bbf62cc4d6cbfa58007fa5e8c09 2024-11-19T12:52:59,734 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/347d0bbf62cc4d6cbfa58007fa5e8c09, entries=150, sequenceid=1112, filesize=12.0 K 2024-11-19T12:52:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/aa19d1625b104a7592d02b35d80fc948 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/aa19d1625b104a7592d02b35d80fc948 2024-11-19T12:52:59,739 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/aa19d1625b104a7592d02b35d80fc948, entries=150, sequenceid=1112, filesize=12.0 K 2024-11-19T12:52:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/ef3241226d38461895f4ba6c906ccde8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ef3241226d38461895f4ba6c906ccde8 2024-11-19T12:52:59,745 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ef3241226d38461895f4ba6c906ccde8, entries=150, sequenceid=1112, filesize=12.0 K 2024-11-19T12:52:59,747 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 8451cbb661c52001cbf3306ac11815b1 in 880ms, sequenceid=1112, compaction requested=false 2024-11-19T12:52:59,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8451cbb661c52001cbf3306ac11815b1: 2024-11-19T12:52:59,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:52:59,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T12:52:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T12:52:59,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T12:52:59,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4940 sec 2024-11-19T12:52:59,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5000 sec 2024-11-19T12:53:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T12:53:00,412 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:00,413 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-19T12:53:00,413 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:00,413 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,413 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:00,423 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-19T12:53:00,423 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:00,423 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,423 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:00,424 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-19T12:53:00,424 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:00,424 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,424 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 233 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 186 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 206 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 257 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3159 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3222 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3184 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3134 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3200 2024-11-19T12:53:00,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:53:00,424 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:53:00,424 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:53:00,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:00,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:00,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:00,425 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:53:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:00,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020780428"}]},"ts":"1732020780428"} 2024-11-19T12:53:00,430 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:53:00,430 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:53:00,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:53:00,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, UNASSIGN}] 2024-11-19T12:53:00,433 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, UNASSIGN 2024-11-19T12:53:00,434 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8451cbb661c52001cbf3306ac11815b1, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:00,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, UNASSIGN because future has completed 2024-11-19T12:53:00,436 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:53:00,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:00,589 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:53:00,589 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:53:00,589 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 8451cbb661c52001cbf3306ac11815b1, disabling compactions & flushes 2024-11-19T12:53:00,589 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:53:00,589 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. after waiting 0 ms 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:53:00,590 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 8451cbb661c52001cbf3306ac11815b1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=A 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=B 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8451cbb661c52001cbf3306ac11815b1, store=C 2024-11-19T12:53:00,590 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:00,593 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b0adea8f8cdc416cb843eb0018c0ecfe is 50, key is test_row_1/A:col10/1732020779562/Put/seqid=0 2024-11-19T12:53:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742156_1332 (size=9857) 2024-11-19T12:53:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:00,999 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1122 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b0adea8f8cdc416cb843eb0018c0ecfe 2024-11-19T12:53:01,007 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/341227db0fb1482a8a6cfbdb7e5a75d3 is 50, key is test_row_1/B:col10/1732020779562/Put/seqid=0 2024-11-19T12:53:01,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742157_1333 (size=9857) 2024-11-19T12:53:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:01,411 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1122 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/341227db0fb1482a8a6cfbdb7e5a75d3 2024-11-19T12:53:01,420 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d0be2beb972e4b1c944fce089289ae70 is 50, key is test_row_1/C:col10/1732020779562/Put/seqid=0 2024-11-19T12:53:01,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742158_1334 (size=9857) 2024-11-19T12:53:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:01,825 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1122 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d0be2beb972e4b1c944fce089289ae70 2024-11-19T12:53:01,829 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/A/b0adea8f8cdc416cb843eb0018c0ecfe as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b0adea8f8cdc416cb843eb0018c0ecfe 2024-11-19T12:53:01,833 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b0adea8f8cdc416cb843eb0018c0ecfe, entries=100, sequenceid=1122, filesize=9.6 K 2024-11-19T12:53:01,834 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/B/341227db0fb1482a8a6cfbdb7e5a75d3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/341227db0fb1482a8a6cfbdb7e5a75d3 2024-11-19T12:53:01,838 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/341227db0fb1482a8a6cfbdb7e5a75d3, entries=100, sequenceid=1122, filesize=9.6 K 2024-11-19T12:53:01,839 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/.tmp/C/d0be2beb972e4b1c944fce089289ae70 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d0be2beb972e4b1c944fce089289ae70 2024-11-19T12:53:01,844 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d0be2beb972e4b1c944fce089289ae70, entries=100, sequenceid=1122, filesize=9.6 K 2024-11-19T12:53:01,844 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8451cbb661c52001cbf3306ac11815b1 in 1254ms, sequenceid=1122, compaction requested=true 2024-11-19T12:53:01,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/807b332108234c639fdcc75b00b89202, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1edcf4d3ad8e48c7a97112da20653944, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/938b7ba3c8464edda7403b6094a4c2ba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a0cb170cccbb4682a08e7f921b1e71e4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3c218ea1ff3b40658a308f6f6d2b8cba, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2ed2551c59314fa0a4f588b356075959, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f62d0b1e01bf4cd8a4321f55ae9151f5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1e263362c1bc43e5aca4ca3521f11c82, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/e1658bf114144a44b22358da1ca600d5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59] to archive 2024-11-19T12:53:01,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:01,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/877263a981254a06b13066a7eeb5e86f 2024-11-19T12:53:01,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/c8642ba7195e4563be11093ebe6e46b8 2024-11-19T12:53:01,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/424d9247cce343b78bb3f4ddabf50b8a 2024-11-19T12:53:01,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/807b332108234c639fdcc75b00b89202 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/807b332108234c639fdcc75b00b89202 2024-11-19T12:53:01,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/135b8955db7f4b9383962ddd503cb9b8 2024-11-19T12:53:01,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2341d736c5bf43b2b4dadd947410fcd7 2024-11-19T12:53:01,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1edcf4d3ad8e48c7a97112da20653944 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1edcf4d3ad8e48c7a97112da20653944 2024-11-19T12:53:01,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/38af8c2ac30d40a081472a82f005769a 2024-11-19T12:53:01,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a76c6efa507c467d90b3f6cf70161eb5 2024-11-19T12:53:01,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1b85dec835d94fefa77b941ad709d5a8 2024-11-19T12:53:01,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/938b7ba3c8464edda7403b6094a4c2ba to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/938b7ba3c8464edda7403b6094a4c2ba 2024-11-19T12:53:01,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3cdec09bb189446589bdf83e789d7c8b 2024-11-19T12:53:01,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a0cb170cccbb4682a08e7f921b1e71e4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a0cb170cccbb4682a08e7f921b1e71e4 2024-11-19T12:53:01,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2dd65ce8b3504dd1af9c5359786e7c92 2024-11-19T12:53:01,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0e674857b5a347268217bca426d8c578 2024-11-19T12:53:01,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/996b71ec1f604d5fb6b224d56067f6e7 2024-11-19T12:53:01,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3c218ea1ff3b40658a308f6f6d2b8cba to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/3c218ea1ff3b40658a308f6f6d2b8cba 2024-11-19T12:53:01,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1d857781ffe845c8bf379e6d6774b997 2024-11-19T12:53:01,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/bb08f7cb088c4cad8389d7b15fb17230 2024-11-19T12:53:01,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2ed2551c59314fa0a4f588b356075959 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2ed2551c59314fa0a4f588b356075959 2024-11-19T12:53:01,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b932f9a9f8cc4a43985f80d498c15e95 2024-11-19T12:53:01,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f62d0b1e01bf4cd8a4321f55ae9151f5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f62d0b1e01bf4cd8a4321f55ae9151f5 2024-11-19T12:53:01,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0be2bbfd5a38440fb11f8d5fd4aab271 2024-11-19T12:53:01,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/370bea07dc4f47c8a2148b81dfdc817d 2024-11-19T12:53:01,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a4ef4862842a4ff8889c9a3f4ed6a82b 2024-11-19T12:53:01,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/a28dd669d45d4ecf972e4eb69ea498ff 2024-11-19T12:53:01,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1e263362c1bc43e5aca4ca3521f11c82 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/1e263362c1bc43e5aca4ca3521f11c82 2024-11-19T12:53:01,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/f04a54d8aeda4edd81df11bff0e3ab83 2024-11-19T12:53:01,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/e1658bf114144a44b22358da1ca600d5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/e1658bf114144a44b22358da1ca600d5 2024-11-19T12:53:01,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/8e9a6347ad874bb19322ac98898a7eb6 2024-11-19T12:53:01,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/0d7259dc5d6543e8a00ebac842f679dd 2024-11-19T12:53:01,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/2976385553d94703805c5dd4baa30e59 2024-11-19T12:53:01,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/f330baadcde540f48c4e7de4c7d07e98, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/4483cb6fd06a4e6891e0ad7f39786b30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0ae530afeef842c29523b2163fedcb29, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/001f342b6c004fc5b195673930a2fe70, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1a4d23946e5f4d938b3c4ba5f776f97a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9b876b3e51a8412ebf18e2b93d2a6ca0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ebdc85e1ebfc4626a665f3f103a0ac4e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ca6771da21594412b6599526df21e6bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/67cc74f4e7884bf8b5dcbf40f82a282f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c] to archive 2024-11-19T12:53:01,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:01,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d3b127db58843d28488a092fad8566c 2024-11-19T12:53:01,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/19a4d6aace324970b2eec654ab240ba5 2024-11-19T12:53:01,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/f330baadcde540f48c4e7de4c7d07e98 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/f330baadcde540f48c4e7de4c7d07e98 2024-11-19T12:53:01,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/050ba5ef8f3c4ff0aa912475bce5ce2e 2024-11-19T12:53:01,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2bc5467872b04e2bb7d15f3f9e2a53d7 2024-11-19T12:53:01,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/4483cb6fd06a4e6891e0ad7f39786b30 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/4483cb6fd06a4e6891e0ad7f39786b30 2024-11-19T12:53:01,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/452566529e1e4c92939b29a9308d275e 2024-11-19T12:53:01,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1ea24113afe5450081d2e7ac795c44e1 2024-11-19T12:53:01,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a05c604bf9bd414f9a84cf55bc55b956 2024-11-19T12:53:01,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0ae530afeef842c29523b2163fedcb29 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0ae530afeef842c29523b2163fedcb29 2024-11-19T12:53:01,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ff00fef2a4da49cfbbab8d000fa760b0 2024-11-19T12:53:01,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/65eae8c85c0a44999cc0ae4bd701ef68 2024-11-19T12:53:01,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/001f342b6c004fc5b195673930a2fe70 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/001f342b6c004fc5b195673930a2fe70 2024-11-19T12:53:01,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8666afc248a44bf8839d927dde302359 2024-11-19T12:53:01,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d1674553119d4f4f9de7f2019dfd4d10 2024-11-19T12:53:01,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1a4d23946e5f4d938b3c4ba5f776f97a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/1a4d23946e5f4d938b3c4ba5f776f97a 2024-11-19T12:53:01,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2d61ba44db374350b95abe6500c9fdcf 2024-11-19T12:53:01,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/62d20695419f402f89bd82f4a6f7641d 2024-11-19T12:53:01,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9b876b3e51a8412ebf18e2b93d2a6ca0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9b876b3e51a8412ebf18e2b93d2a6ca0 2024-11-19T12:53:01,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/2cb7e083b68b4d3b9f4b7efe2eb30fc5 2024-11-19T12:53:01,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/d049e4c2585b4b41acf7274b41173ea6 2024-11-19T12:53:01,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ebdc85e1ebfc4626a665f3f103a0ac4e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ebdc85e1ebfc4626a665f3f103a0ac4e 2024-11-19T12:53:01,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/7859ec917b7a4edd8539fae65c145cb1 2024-11-19T12:53:01,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0bb0448d217649baa0cf8644fbb2fa41 2024-11-19T12:53:01,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/a7344792d43f4c57b375c6520894ad11 2024-11-19T12:53:01,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ca6771da21594412b6599526df21e6bf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/ca6771da21594412b6599526df21e6bf 2024-11-19T12:53:01,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/9d28ece2bb6444bf9c5aee8319ea0e96 2024-11-19T12:53:01,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/649a8889a51b45e38995157e2dde8635 2024-11-19T12:53:01,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/67cc74f4e7884bf8b5dcbf40f82a282f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/67cc74f4e7884bf8b5dcbf40f82a282f 2024-11-19T12:53:01,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/0b07fb447f474137a9e3e73717cdd050 2024-11-19T12:53:01,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8d3f978455e24415bfabc01e23592cd5 2024-11-19T12:53:01,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/91862f998ccf48c08a81c706227d635c 2024-11-19T12:53:01,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e75816481af14b7db7f4a7a609a227b3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/1b6ccb1832f7485eafe047d58fa72b2a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ff3b47fdad414dc8baf811738026ca6b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/98a4e467ed584c85b9adf746a8f22d9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/0da7cc98f95b4bdda0feb8bf251a005f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/40d17e657fa64410b01fbbd9d86523c1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/65eadf51bc444e79891a320ce4585dc9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a058252145654c9a9f960d7c841ece2b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ead8ca294f7b48949d75b6235e40f1fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47] to archive 2024-11-19T12:53:01,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:01,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/92665b922e524768aa946b6ab60a5fe9 2024-11-19T12:53:01,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/95d97e5008124c1bbe34e7a4a11f30d8 2024-11-19T12:53:01,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e75816481af14b7db7f4a7a609a227b3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e75816481af14b7db7f4a7a609a227b3 2024-11-19T12:53:01,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/7f0b078b720b448fb3c24c71daad7114 2024-11-19T12:53:01,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/62f13352c62d4511bb2d532042664bee 2024-11-19T12:53:01,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/1b6ccb1832f7485eafe047d58fa72b2a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/1b6ccb1832f7485eafe047d58fa72b2a 2024-11-19T12:53:01,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/49e6493b5f0b47f8b28bd8187ae17640 2024-11-19T12:53:01,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c11a2a98baf44e10bf795f60ba811d6a 2024-11-19T12:53:01,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/08c76c59c534456b820904b63eb1aa35 2024-11-19T12:53:01,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ff3b47fdad414dc8baf811738026ca6b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ff3b47fdad414dc8baf811738026ca6b 2024-11-19T12:53:01,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/da315eacff314327a8cdbec04a7d72a4 2024-11-19T12:53:01,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/acdf051309344008b377b2826944cc36 2024-11-19T12:53:01,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/98a4e467ed584c85b9adf746a8f22d9a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/98a4e467ed584c85b9adf746a8f22d9a 2024-11-19T12:53:01,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/c7f5d04bc4904cc59eb395dfd4361ba2 2024-11-19T12:53:01,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/f727094b80bc41d09dbcc9fd19979c80 2024-11-19T12:53:01,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/0da7cc98f95b4bdda0feb8bf251a005f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/0da7cc98f95b4bdda0feb8bf251a005f 2024-11-19T12:53:01,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/fb34c79a92c8437bb42ad4d5a435a8d0 2024-11-19T12:53:01,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/26c60c306e7344428619541680b44f17 2024-11-19T12:53:01,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/40d17e657fa64410b01fbbd9d86523c1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/40d17e657fa64410b01fbbd9d86523c1 2024-11-19T12:53:01,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/74bbca0fd0db4bc099275331b9c6dea5 2024-11-19T12:53:01,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a8cc30f01f67472a9604f9213646d1fe 2024-11-19T12:53:01,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/65eadf51bc444e79891a320ce4585dc9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/65eadf51bc444e79891a320ce4585dc9 2024-11-19T12:53:01,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/09247815ccec4ab99b450a806539064d 2024-11-19T12:53:01,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/b25028890f5a499989989e27fef1b2bf 2024-11-19T12:53:01,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ad191da173354255ab774af474b7b39a 2024-11-19T12:53:01,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a058252145654c9a9f960d7c841ece2b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a058252145654c9a9f960d7c841ece2b 2024-11-19T12:53:01,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/e4432b2a955a44f68770651be013c290 2024-11-19T12:53:01,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ab0dc2c699004dcfb9bce566a68f77df 2024-11-19T12:53:02,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ead8ca294f7b48949d75b6235e40f1fe to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ead8ca294f7b48949d75b6235e40f1fe 2024-11-19T12:53:02,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d567a8b9a2334e52af594dd7fe4b8f19 2024-11-19T12:53:02,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/648ab6866aae46cb85bd68535b618136 2024-11-19T12:53:02,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/54904abecf064395bb02991bce627b47 2024-11-19T12:53:02,010 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/recovered.edits/1125.seqid, newMaxSeqId=1125, maxSeqId=1 2024-11-19T12:53:02,010 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. 2024-11-19T12:53:02,011 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 8451cbb661c52001cbf3306ac11815b1: Waiting for close lock at 1732020780589Running coprocessor pre-close hooks at 1732020780589Disabling compacts and flushes for region at 1732020780589Disabling writes for close at 1732020780590 (+1 ms)Obtaining lock to block concurrent updates at 1732020780590Preparing flush snapshotting stores in 8451cbb661c52001cbf3306ac11815b1 at 1732020780590Finished memstore snapshotting TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732020780590Flushing stores of TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1. at 1732020780590Flushing 8451cbb661c52001cbf3306ac11815b1/A: creating writer at 1732020780591 (+1 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/A: appending metadata at 1732020780593 (+2 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/A: closing flushed file at 1732020780593Flushing 8451cbb661c52001cbf3306ac11815b1/B: creating writer at 1732020781005 (+412 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/B: appending metadata at 1732020781007 (+2 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/B: closing flushed file at 1732020781007Flushing 8451cbb661c52001cbf3306ac11815b1/C: creating writer at 1732020781417 (+410 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/C: appending metadata at 1732020781420 (+3 ms)Flushing 8451cbb661c52001cbf3306ac11815b1/C: closing flushed file at 1732020781420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c7ae21a: reopening flushed file at 1732020781828 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74a3b3d: reopening flushed file at 1732020781833 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@316ffe64: reopening flushed file at 1732020781838 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8451cbb661c52001cbf3306ac11815b1 in 1254ms, sequenceid=1122, compaction requested=true at 1732020781844 (+6 ms)Writing region close event to WAL at 1732020782006 (+162 ms)Running coprocessor post-close hooks at 1732020782010 (+4 ms)Closed at 1732020782010 2024-11-19T12:53:02,013 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:53:02,013 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8451cbb661c52001cbf3306ac11815b1, regionState=CLOSED 2024-11-19T12:53:02,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:02,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T12:53:02,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 8451cbb661c52001cbf3306ac11815b1, server=aba5a916dfea,35349,1732020694570 in 1.5800 sec 2024-11-19T12:53:02,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-19T12:53:02,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8451cbb661c52001cbf3306ac11815b1, UNASSIGN in 1.5860 sec 2024-11-19T12:53:02,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T12:53:02,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-11-19T12:53:02,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020782023"}]},"ts":"1732020782023"} 2024-11-19T12:53:02,024 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:53:02,024 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:53:02,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6000 sec 2024-11-19T12:53:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T12:53:02,571 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:02,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:53:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,574 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T12:53:02,575 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,577 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:53:02,581 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/recovered.edits] 2024-11-19T12:53:02,587 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/347d0bbf62cc4d6cbfa58007fa5e8c09 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/347d0bbf62cc4d6cbfa58007fa5e8c09 2024-11-19T12:53:02,589 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/47989bbd230d4b1494b8850cb96c117d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/47989bbd230d4b1494b8850cb96c117d 2024-11-19T12:53:02,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b0adea8f8cdc416cb843eb0018c0ecfe to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/A/b0adea8f8cdc416cb843eb0018c0ecfe 2024-11-19T12:53:02,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/341227db0fb1482a8a6cfbdb7e5a75d3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/341227db0fb1482a8a6cfbdb7e5a75d3 2024-11-19T12:53:02,597 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8a685562b67d4b40a40cd1fe026210f5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/8a685562b67d4b40a40cd1fe026210f5 2024-11-19T12:53:02,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/aa19d1625b104a7592d02b35d80fc948 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/B/aa19d1625b104a7592d02b35d80fc948 2024-11-19T12:53:02,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a6bac24037794888bb892e28e30b7699 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/a6bac24037794888bb892e28e30b7699 2024-11-19T12:53:02,627 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d0be2beb972e4b1c944fce089289ae70 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/d0be2beb972e4b1c944fce089289ae70 2024-11-19T12:53:02,629 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ef3241226d38461895f4ba6c906ccde8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/C/ef3241226d38461895f4ba6c906ccde8 2024-11-19T12:53:02,634 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/recovered.edits/1125.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1/recovered.edits/1125.seqid 2024-11-19T12:53:02,635 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/8451cbb661c52001cbf3306ac11815b1 2024-11-19T12:53:02,635 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:53:02,637 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,642 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:53:02,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:53:02,646 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,646 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:53:02,646 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020782646"}]},"ts":"9223372036854775807"} 2024-11-19T12:53:02,648 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:53:02,648 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8451cbb661c52001cbf3306ac11815b1, NAME => 'TestAcidGuarantees,,1732020756670.8451cbb661c52001cbf3306ac11815b1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:53:02,648 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:53:02,648 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020782648"}]},"ts":"9223372036854775807"} 2024-11-19T12:53:02,651 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:53:02,652 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-19T12:53:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T12:53:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:53:02,682 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:02,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 252), OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=424 (was 370) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5614 (was 5811) 2024-11-19T12:53:02,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=5616 2024-11-19T12:53:02,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:53:02,707 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:53:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:02,710 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:53:02,710 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:02,710 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-19T12:53:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:02,711 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:53:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742159_1335 (size=1061) 2024-11-19T12:53:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:02,958 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:53:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:03,118 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:53:03,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742160_1336 (size=53) 2024-11-19T12:53:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:03,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:53:03,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 61dc078310d7baf1f63658f1d6fdfff1, disabling compactions & flushes 2024-11-19T12:53:03,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. after waiting 0 ms 2024-11-19T12:53:03,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 61dc078310d7baf1f63658f1d6fdfff1: Waiting for close lock at 1732020783524Disabling compacts and flushes for region at 1732020783524Disabling writes for close at 1732020783525 (+1 ms)Writing region close event to WAL at 1732020783525Closed at 1732020783525 2024-11-19T12:53:03,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:53:03,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020783526"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020783526"}]},"ts":"1732020783526"} 2024-11-19T12:53:03,528 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:53:03,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:53:03,529 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020783529"}]},"ts":"1732020783529"} 2024-11-19T12:53:03,530 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:53:03,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, ASSIGN}] 2024-11-19T12:53:03,531 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, ASSIGN 2024-11-19T12:53:03,532 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:53:03,683 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:03,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, ASSIGN because future has completed 2024-11-19T12:53:03,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:03,839 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:53:03,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:53:03,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,839 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,841 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,842 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:03,842 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName A 2024-11-19T12:53:03,842 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:03,843 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:03,843 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,844 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:03,844 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName B 2024-11-19T12:53:03,844 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:03,844 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:03,844 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,845 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:03,845 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName C 2024-11-19T12:53:03,845 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:03,846 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:03,846 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,846 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,847 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,847 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,848 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,848 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,848 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:53:03,849 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:03,851 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:53:03,852 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 61dc078310d7baf1f63658f1d6fdfff1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73593188, jitterRate=0.09662395715713501}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:53:03,852 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:03,852 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 61dc078310d7baf1f63658f1d6fdfff1: Running coprocessor pre-open hook at 1732020783839Writing region info on filesystem at 1732020783839Initializing all the Stores at 1732020783840 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020783840Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020783840Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020783840Cleaning up temporary data from old regions at 1732020783848 (+8 ms)Running coprocessor post-open hooks at 1732020783852 (+4 ms)Region opened successfully at 1732020783852 2024-11-19T12:53:03,853 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., pid=87, masterSystemTime=1732020783836 2024-11-19T12:53:03,855 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,855 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:03,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:03,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:03,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T12:53:03,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 in 174 msec 2024-11-19T12:53:03,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-19T12:53:03,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, ASSIGN in 330 msec 2024-11-19T12:53:03,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:53:03,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020783864"}]},"ts":"1732020783864"} 2024-11-19T12:53:03,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:53:03,867 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:53:03,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-19T12:53:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-19T12:53:04,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:04,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:04,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:04,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:04,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:04,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4951e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:04,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0d445b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:04,867 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:04,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:04,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:04,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:53:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:53:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b8725c2 2024-11-19T12:53:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:53:04,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:53:04,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:53:04,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:53:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742161_1337 (size=1097) 2024-11-19T12:53:05,287 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T12:53:05,287 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T12:53:05,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:53:05,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE}] 2024-11-19T12:53:05,293 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE 2024-11-19T12:53:05,293 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:05,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE because future has completed 2024-11-19T12:53:05,295 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:53:05,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:05,447 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,447 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:53:05,447 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 61dc078310d7baf1f63658f1d6fdfff1, disabling compactions & flushes 2024-11-19T12:53:05,447 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,447 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,447 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. after waiting 0 ms 2024-11-19T12:53:05,447 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,450 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:53:05,451 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,451 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 61dc078310d7baf1f63658f1d6fdfff1: Waiting for close lock at 1732020785447Running coprocessor pre-close hooks at 1732020785447Disabling compacts and flushes for region at 1732020785447Disabling writes for close at 1732020785447Writing region close event to WAL at 1732020785448 (+1 ms)Running coprocessor post-close hooks at 1732020785451 (+3 ms)Closed at 1732020785451 2024-11-19T12:53:05,451 WARN [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 61dc078310d7baf1f63658f1d6fdfff1 to self. 2024-11-19T12:53:05,453 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,453 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=CLOSED 2024-11-19T12:53:05,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:05,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T12:53:05,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 in 160 msec 2024-11-19T12:53:05,457 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE; state=CLOSED, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=true 2024-11-19T12:53:05,608 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:05,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE because future has completed 2024-11-19T12:53:05,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:05,764 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,765 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:53:05,765 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,765 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:53:05,765 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,765 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,766 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,767 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:05,767 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName A 2024-11-19T12:53:05,768 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:05,769 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:05,769 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,770 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:05,770 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName B 2024-11-19T12:53:05,770 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:05,770 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:05,771 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,771 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:05,771 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61dc078310d7baf1f63658f1d6fdfff1 columnFamilyName C 2024-11-19T12:53:05,771 DEBUG [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:05,772 INFO [StoreOpener-61dc078310d7baf1f63658f1d6fdfff1-1 {}] regionserver.HStore(327): Store=61dc078310d7baf1f63658f1d6fdfff1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:05,772 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,772 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,773 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,774 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,775 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,775 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,775 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:53:05,776 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,777 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 61dc078310d7baf1f63658f1d6fdfff1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67773082, jitterRate=0.009897619485855103}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:53:05,777 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,777 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 61dc078310d7baf1f63658f1d6fdfff1: Running coprocessor pre-open hook at 1732020785765Writing region info on filesystem at 1732020785765Initializing all the Stores at 1732020785766 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020785766Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020785766Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020785766Cleaning up temporary data from old regions at 1732020785775 (+9 ms)Running coprocessor post-open hooks at 1732020785777 (+2 ms)Region opened successfully at 1732020785777 2024-11-19T12:53:05,778 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., pid=92, masterSystemTime=1732020785761 2024-11-19T12:53:05,780 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,780 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:05,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=OPEN, openSeqNum=5, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:05,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:05,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-19T12:53:05,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 in 173 msec 2024-11-19T12:53:05,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T12:53:05,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, REOPEN/MOVE in 492 msec 2024-11-19T12:53:05,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T12:53:05,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-11-19T12:53:05,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 914 msec 2024-11-19T12:53:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T12:53:05,792 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:05,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a862268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138f368b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,798 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2060870c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd4f213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,802 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8e8383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf97e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5086db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c866ad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cfaf4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706fbeb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aabd1f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299edc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,818 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516adaa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,820 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1878f484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,822 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a3bb4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,824 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe18ab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aadf16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,828 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f35967e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:05,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:05,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f204523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:05,833 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5b2453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:05,834 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,834 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,834 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,844 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,845 DEBUG [Thread-1553 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,845 DEBUG [Thread-1553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,851 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,863 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:53:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:05,875 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,894 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,918 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,932 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:05,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:53:05,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-19T12:53:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:05,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:05,945 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:05,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:05,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d6dddf23164f4a199ff776123b6180ee_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020785847/Put/seqid=0 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020785932, max timestamp=1732020785969Num uniques 859; 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020785932, max timestamp=1732020785969Num uniques 859; 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020785932, max timestamp=1732020785969Num uniques 859; 2024-11-19T12:53:05,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:05,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742162_1338 (size=14594) 2024-11-19T12:53:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020846009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020846010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020846021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020846023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020846025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:06,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020846131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020846141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020846141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020846152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020846152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:06,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020846352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020846352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020846352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:06,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020846371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020846371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d6dddf23164f4a199ff776123b6180ee_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6dddf23164f4a199ff776123b6180ee_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:06,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/449aef31ae054f7b8f008ab4aa663309, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:06,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/449aef31ae054f7b8f008ab4aa663309 is 175, key is test_row_0/A:col10/1732020785847/Put/seqid=0 2024-11-19T12:53:06,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742163_1339 (size=39549) 2024-11-19T12:53:06,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:06,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020846672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020846673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020846673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020846693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020846693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:06,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,832 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/449aef31ae054f7b8f008ab4aa663309 2024-11-19T12:53:06,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:06,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:06,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8adabcd6a5f54672a421029a2607eb41 is 50, key is test_row_0/B:col10/1732020785847/Put/seqid=0 2024-11-19T12:53:06,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742164_1340 (size=12001) 2024-11-19T12:53:06,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8adabcd6a5f54672a421029a2607eb41 2024-11-19T12:53:07,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e7083807fd38440eb415ffbb4ee14725 is 50, key is test_row_0/C:col10/1732020785847/Put/seqid=0 2024-11-19T12:53:07,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742165_1341 (size=12001) 2024-11-19T12:53:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:07,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020847181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020847181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020847182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:07,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020847203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:07,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020847211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:07,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e7083807fd38440eb415ffbb4ee14725 2024-11-19T12:53:07,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/449aef31ae054f7b8f008ab4aa663309 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309 2024-11-19T12:53:07,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309, entries=200, sequenceid=14, filesize=38.6 K 2024-11-19T12:53:07,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:07,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8adabcd6a5f54672a421029a2607eb41 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41 2024-11-19T12:53:07,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T12:53:07,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e7083807fd38440eb415ffbb4ee14725 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725 2024-11-19T12:53:07,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T12:53:07,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 61dc078310d7baf1f63658f1d6fdfff1 in 1660ms, sequenceid=14, compaction requested=false 2024-11-19T12:53:07,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:07,577 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:53:07,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T12:53:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:07,646 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T12:53:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d1f866b9b324250b3233aeb9a9dc8a7_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020786002/Put/seqid=0 2024-11-19T12:53:07,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742166_1342 (size=12154) 2024-11-19T12:53:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:08,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:08,166 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d1f866b9b324250b3233aeb9a9dc8a7_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d1f866b9b324250b3233aeb9a9dc8a7_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:08,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/248e6e561b9345f2bde464f493f43543, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/248e6e561b9345f2bde464f493f43543 is 175, key is test_row_0/A:col10/1732020786002/Put/seqid=0 2024-11-19T12:53:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:08,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:08,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742167_1343 (size=30955) 2024-11-19T12:53:08,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020848230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020848231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020848231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020848231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,247 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020848233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020848351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020848351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020848352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020848362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020848562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020848562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020848563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020848571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,633 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/248e6e561b9345f2bde464f493f43543 2024-11-19T12:53:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/aa23780b991044c28c6b253f1cc05ee3 is 50, key is test_row_0/B:col10/1732020786002/Put/seqid=0 2024-11-19T12:53:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742168_1344 (size=12001) 2024-11-19T12:53:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020848881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020848881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020848881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020848881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,117 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/aa23780b991044c28c6b253f1cc05ee3 2024-11-19T12:53:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5a4b184d212940cbaffef61bacb30da7 is 50, key is test_row_0/C:col10/1732020786002/Put/seqid=0 2024-11-19T12:53:09,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742169_1345 (size=12001) 2024-11-19T12:53:09,132 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5a4b184d212940cbaffef61bacb30da7 2024-11-19T12:53:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/248e6e561b9345f2bde464f493f43543 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543 2024-11-19T12:53:09,149 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543, entries=150, sequenceid=49, filesize=30.2 K 2024-11-19T12:53:09,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/aa23780b991044c28c6b253f1cc05ee3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3 2024-11-19T12:53:09,154 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T12:53:09,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5a4b184d212940cbaffef61bacb30da7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7 2024-11-19T12:53:09,160 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T12:53:09,161 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 61dc078310d7baf1f63658f1d6fdfff1 in 1515ms, sequenceid=49, compaction requested=false 2024-11-19T12:53:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:09,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-19T12:53:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-19T12:53:09,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T12:53:09,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2220 sec 2024-11-19T12:53:09,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.2280 sec 2024-11-19T12:53:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:09,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:09,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:09,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191cf92b6b221c4fd8b53a936d3a8fe6ba_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742170_1346 (size=14594) 2024-11-19T12:53:09,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020849442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020789429, max timestamp=1732020789453Num uniques 859; 2024-11-19T12:53:09,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020789429, max timestamp=1732020789453Num uniques 859; 2024-11-19T12:53:09,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:09,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:09,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020789429, max timestamp=1732020789453Num uniques 859; 2024-11-19T12:53:09,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:09,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020849502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020849508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020849509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020849561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020849621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020849622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020849622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020849781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,847 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:09,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020849841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191cf92b6b221c4fd8b53a936d3a8fe6ba_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cf92b6b221c4fd8b53a936d3a8fe6ba_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020849841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020849841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:09,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/f1d555ed6d1a4a979ba1a6e457415349, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:09,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/f1d555ed6d1a4a979ba1a6e457415349 is 175, key is test_row_0/A:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:09,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742171_1347 (size=39549) 2024-11-19T12:53:09,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/f1d555ed6d1a4a979ba1a6e457415349 2024-11-19T12:53:09,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3b76e12d534b48378fba100ddb7a4787 is 50, key is test_row_0/B:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:09,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742172_1348 (size=12001) 2024-11-19T12:53:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020850101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T12:53:10,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:10,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T12:53:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:10,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:10,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:10,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020850161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020850162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020850162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:10,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T12:53:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020850261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,269 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3b76e12d534b48378fba100ddb7a4787 2024-11-19T12:53:10,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f78647a406da47588eab4970bdf01086 is 50, key is test_row_0/C:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:10,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742173_1349 (size=12001) 2024-11-19T12:53:10,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T12:53:10,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:10,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:10,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T12:53:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020850611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f78647a406da47588eab4970bdf01086 2024-11-19T12:53:10,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/f1d555ed6d1a4a979ba1a6e457415349 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349 2024-11-19T12:53:10,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349, entries=200, sequenceid=60, filesize=38.6 K 2024-11-19T12:53:10,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3b76e12d534b48378fba100ddb7a4787 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787 2024-11-19T12:53:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020850681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020850681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020850681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:10,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T12:53:10,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f78647a406da47588eab4970bdf01086 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086 2024-11-19T12:53:10,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T12:53:10,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 61dc078310d7baf1f63658f1d6fdfff1 in 1313ms, sequenceid=60, compaction requested=true 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:10,709 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T12:53:10,709 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:10,710 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:10,710 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:10,710 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:10,710 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:10,710 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,710 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,710 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.2 K 2024-11-19T12:53:10,710 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=107.5 K 2024-11-19T12:53:10,710 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,710 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349] 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8adabcd6a5f54672a421029a2607eb41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020785847 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449aef31ae054f7b8f008ab4aa663309, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020785847 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting aa23780b991044c28c6b253f1cc05ee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732020786002 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 248e6e561b9345f2bde464f493f43543, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732020786002 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b76e12d534b48378fba100ddb7a4787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:10,711 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d555ed6d1a4a979ba1a6e457415349, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:10,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,724 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:10,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:10,735 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:10,736 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/c0062c572c7949d3ba8d849057cb185a is 50, key is test_row_0/B:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:10,739 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:10,758 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b8409a7a0bf14b08b3458f15f80f121f_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:10,761 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b8409a7a0bf14b08b3458f15f80f121f_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:10,761 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8409a7a0bf14b08b3458f15f80f121f_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:10,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119edd7405d1d894d2f8587c51c9e639a85_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020789477/Put/seqid=0 2024-11-19T12:53:10,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742174_1350 (size=12104) 2024-11-19T12:53:10,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/c0062c572c7949d3ba8d849057cb185a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c0062c572c7949d3ba8d849057cb185a 2024-11-19T12:53:10,777 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into c0062c572c7949d3ba8d849057cb185a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:10,777 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:10,777 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=13, startTime=1732020790709; duration=0sec 2024-11-19T12:53:10,777 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:10,777 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:10,777 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:10,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742176_1352 (size=12154) 2024-11-19T12:53:10,780 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:10,780 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:10,780 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:10,780 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.2 K 2024-11-19T12:53:10,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742175_1351 (size=4469) 2024-11-19T12:53:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:10,781 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e7083807fd38440eb415ffbb4ee14725, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732020785847 2024-11-19T12:53:10,782 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4b184d212940cbaffef61bacb30da7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732020786002 2024-11-19T12:53:10,782 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#303 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:10,783 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ed71091db78e4d66abfd95358cd162ae is 175, key is test_row_0/A:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:10,784 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f78647a406da47588eab4970bdf01086, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:10,787 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119edd7405d1d894d2f8587c51c9e639a85_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119edd7405d1d894d2f8587c51c9e639a85_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b45707d02b5c4a069137f987b21cc6f3, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b45707d02b5c4a069137f987b21cc6f3 is 175, key is test_row_0/A:col10/1732020789477/Put/seqid=0 2024-11-19T12:53:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742177_1353 (size=31058) 2024-11-19T12:53:10,811 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ed71091db78e4d66abfd95358cd162ae as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae 2024-11-19T12:53:10,818 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:10,818 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/886c5d7f8efd4c34850c3fba21e9091e is 50, key is test_row_0/C:col10/1732020788231/Put/seqid=0 2024-11-19T12:53:10,823 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into ed71091db78e4d66abfd95358cd162ae(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:10,823 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:10,823 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=13, startTime=1732020790709; duration=0sec 2024-11-19T12:53:10,823 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:10,823 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:10,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742178_1354 (size=30955) 2024-11-19T12:53:10,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742179_1355 (size=12104) 2024-11-19T12:53:11,232 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b45707d02b5c4a069137f987b21cc6f3 2024-11-19T12:53:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/7aa9cb3fd0554c2c90e01697484b7812 is 50, key is test_row_0/B:col10/1732020789477/Put/seqid=0 2024-11-19T12:53:11,244 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/886c5d7f8efd4c34850c3fba21e9091e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/886c5d7f8efd4c34850c3fba21e9091e 2024-11-19T12:53:11,250 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into 886c5d7f8efd4c34850c3fba21e9091e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:11,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742180_1356 (size=12001) 2024-11-19T12:53:11,250 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:11,250 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=13, startTime=1732020790709; duration=0sec 2024-11-19T12:53:11,251 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:11,251 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:11,651 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/7aa9cb3fd0554c2c90e01697484b7812 2024-11-19T12:53:11,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5045b9e17c064a1fa2f3ff46b5caf2d7 is 50, key is test_row_0/C:col10/1732020789477/Put/seqid=0 2024-11-19T12:53:11,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742181_1357 (size=12001) 2024-11-19T12:53:11,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020851680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020851701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:11,707 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020851711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020851712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:11,718 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,719 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020851791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020852001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:12,061 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5045b9e17c064a1fa2f3ff46b5caf2d7 2024-11-19T12:53:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b45707d02b5c4a069137f987b21cc6f3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3 2024-11-19T12:53:12,068 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3, entries=150, sequenceid=95, filesize=30.2 K 2024-11-19T12:53:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/7aa9cb3fd0554c2c90e01697484b7812 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812 2024-11-19T12:53:12,073 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T12:53:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/5045b9e17c064a1fa2f3ff46b5caf2d7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7 2024-11-19T12:53:12,078 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T12:53:12,078 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 61dc078310d7baf1f63658f1d6fdfff1 in 1354ms, sequenceid=95, compaction requested=false 2024-11-19T12:53:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:12,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T12:53:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T12:53:12,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T12:53:12,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-19T12:53:12,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9690 sec 2024-11-19T12:53:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T12:53:12,261 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:12,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T12:53:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:12,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:12,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:12,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:12,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:12,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:12,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111924c3e0c6280c4035bf972dc4f49cdeb4_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:12,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742182_1358 (size=14594) 2024-11-19T12:53:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:12,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020852421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020852531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:12,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:12,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:12,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111924c3e0c6280c4035bf972dc4f49cdeb4_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111924c3e0c6280c4035bf972dc4f49cdeb4_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:12,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/bc0e043c79174d7080b6d8062b3e4da5, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:12,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/bc0e043c79174d7080b6d8062b3e4da5 is 175, key is test_row_0/A:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020852741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:12,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742183_1359 (size=39549) 2024-11-19T12:53:12,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:12,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:12,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:13,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020853061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:13,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/bc0e043c79174d7080b6d8062b3e4da5 2024-11-19T12:53:13,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/c412bbadf9de4a17b23b1801df4886ce is 50, key is test_row_0/B:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:13,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742184_1360 (size=12001) 2024-11-19T12:53:13,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:13,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/c412bbadf9de4a17b23b1801df4886ce 2024-11-19T12:53:13,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/2be46b1047774dff9f0e9fc52f7a8549 is 50, key is test_row_0/C:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:13,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742185_1361 (size=12001) 2024-11-19T12:53:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020853582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:13,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:13,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020853731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020853731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020853731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:13,738 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,738 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:13,738 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:13,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:13,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/2be46b1047774dff9f0e9fc52f7a8549 2024-11-19T12:53:13,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/bc0e043c79174d7080b6d8062b3e4da5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5 2024-11-19T12:53:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5, entries=200, sequenceid=110, filesize=38.6 K 2024-11-19T12:53:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/c412bbadf9de4a17b23b1801df4886ce as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce 2024-11-19T12:53:13,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce, entries=150, sequenceid=110, filesize=11.7 K 2024-11-19T12:53:13,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/2be46b1047774dff9f0e9fc52f7a8549 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549 2024-11-19T12:53:13,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549, entries=150, sequenceid=110, filesize=11.7 K 2024-11-19T12:53:13,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 61dc078310d7baf1f63658f1d6fdfff1 in 1672ms, sequenceid=110, compaction requested=true 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:13,991 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:13,991 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:13,992 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,992 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,992 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=99.2 K 2024-11-19T12:53:13,992 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c0062c572c7949d3ba8d849057cb185a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.3 K 2024-11-19T12:53:13,992 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5] 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c0062c572c7949d3ba8d849057cb185a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:13,992 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed71091db78e4d66abfd95358cd162ae, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:13,993 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa9cb3fd0554c2c90e01697484b7812, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732020789477 2024-11-19T12:53:13,993 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45707d02b5c4a069137f987b21cc6f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732020789477 2024-11-19T12:53:13,993 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c412bbadf9de4a17b23b1801df4886ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:13,993 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0e043c79174d7080b6d8062b3e4da5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:13,999 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:14,003 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c4bb944e1a044afd8d3b10cc74ac1003_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:14,004 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c4bb944e1a044afd8d3b10cc74ac1003_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:14,004 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:14,004 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c4bb944e1a044afd8d3b10cc74ac1003_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:14,005 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/4410bb57284b47999ff884880c152010 is 50, key is test_row_0/B:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742186_1362 (size=12207) 2024-11-19T12:53:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742187_1363 (size=4469) 2024-11-19T12:53:14,019 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#311 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:14,020 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/91922b159c06498ca06bd104603bd62a is 175, key is test_row_0/A:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:14,021 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/4410bb57284b47999ff884880c152010 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/4410bb57284b47999ff884880c152010 2024-11-19T12:53:14,027 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into 4410bb57284b47999ff884880c152010(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:14,027 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:14,027 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=13, startTime=1732020793991; duration=0sec 2024-11-19T12:53:14,027 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:14,027 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:14,027 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:14,028 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:14,028 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:14,028 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:14,028 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/886c5d7f8efd4c34850c3fba21e9091e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.3 K 2024-11-19T12:53:14,029 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 886c5d7f8efd4c34850c3fba21e9091e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732020788226 2024-11-19T12:53:14,029 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5045b9e17c064a1fa2f3ff46b5caf2d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732020789477 2024-11-19T12:53:14,029 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be46b1047774dff9f0e9fc52f7a8549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:14,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742188_1364 (size=31161) 2024-11-19T12:53:14,035 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:14,035 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/cf6ee12e56984e4d9c9a8caf017bbcf7 is 50, key is test_row_0/C:col10/1732020792316/Put/seqid=0 2024-11-19T12:53:14,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742189_1365 (size=12207) 2024-11-19T12:53:14,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T12:53:14,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:14,094 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119111884f835f94347bf452e9136b7401b_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020792399/Put/seqid=0 2024-11-19T12:53:14,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742190_1366 (size=12204) 2024-11-19T12:53:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:14,116 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119111884f835f94347bf452e9136b7401b_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119111884f835f94347bf452e9136b7401b_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:14,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b2807054bafa4666acac75a105b8ea19, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:14,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b2807054bafa4666acac75a105b8ea19 is 175, key is test_row_0/A:col10/1732020792399/Put/seqid=0 2024-11-19T12:53:14,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742191_1367 (size=31005) 2024-11-19T12:53:14,123 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b2807054bafa4666acac75a105b8ea19 2024-11-19T12:53:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8e58699b1e9941bf904f42411aac2539 is 50, key is test_row_0/B:col10/1732020792399/Put/seqid=0 2024-11-19T12:53:14,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742192_1368 (size=12051) 2024-11-19T12:53:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:14,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020854381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:14,434 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/91922b159c06498ca06bd104603bd62a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a 2024-11-19T12:53:14,445 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into 91922b159c06498ca06bd104603bd62a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:14,445 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:14,445 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=13, startTime=1732020793991; duration=0sec 2024-11-19T12:53:14,446 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:14,446 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:14,460 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/cf6ee12e56984e4d9c9a8caf017bbcf7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/cf6ee12e56984e4d9c9a8caf017bbcf7 2024-11-19T12:53:14,465 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into cf6ee12e56984e4d9c9a8caf017bbcf7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:14,465 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:14,465 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=13, startTime=1732020793991; duration=0sec 2024-11-19T12:53:14,466 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:14,466 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020854491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:14,543 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8e58699b1e9941bf904f42411aac2539 2024-11-19T12:53:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/ad6088354fbd4a27abf5666727f5420b is 50, key is test_row_0/C:col10/1732020792399/Put/seqid=0 2024-11-19T12:53:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742193_1369 (size=12051) 2024-11-19T12:53:14,575 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/ad6088354fbd4a27abf5666727f5420b 2024-11-19T12:53:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/b2807054bafa4666acac75a105b8ea19 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19 2024-11-19T12:53:14,585 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19, entries=150, sequenceid=131, filesize=30.3 K 2024-11-19T12:53:14,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8e58699b1e9941bf904f42411aac2539 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539 2024-11-19T12:53:14,591 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T12:53:14,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/ad6088354fbd4a27abf5666727f5420b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b 2024-11-19T12:53:14,595 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T12:53:14,596 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 61dc078310d7baf1f63658f1d6fdfff1 in 502ms, sequenceid=131, compaction requested=false 2024-11-19T12:53:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T12:53:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T12:53:14,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T12:53:14,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-19T12:53:14,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.3380 sec 2024-11-19T12:53:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:14,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:14,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ae6c7bcae3f4f999f427a9b759d5225_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:14,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742194_1370 (size=14794) 2024-11-19T12:53:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020854734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020854747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:14,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020854851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:14,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020854863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,018 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:15,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ae6c7bcae3f4f999f427a9b759d5225_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ae6c7bcae3f4f999f427a9b759d5225_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:15,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a2a925377a14461fab9eabf74cdc3656, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:15,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a2a925377a14461fab9eabf74cdc3656 is 175, key is test_row_0/A:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742195_1371 (size=39749) 2024-11-19T12:53:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020855072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020855081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020855391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020855392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a2a925377a14461fab9eabf74cdc3656 2024-11-19T12:53:15,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/913a99f393414922b3962c23e1889cca is 50, key is test_row_0/B:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742196_1372 (size=12151) 2024-11-19T12:53:15,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/913a99f393414922b3962c23e1889cca 2024-11-19T12:53:15,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/37795a227d0b493ba22e1adb25f3cc9f is 50, key is test_row_0/C:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742197_1373 (size=12151) 2024-11-19T12:53:15,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/37795a227d0b493ba22e1adb25f3cc9f 2024-11-19T12:53:15,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a2a925377a14461fab9eabf74cdc3656 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656 2024-11-19T12:53:15,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656, entries=200, sequenceid=146, filesize=38.8 K 2024-11-19T12:53:15,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/913a99f393414922b3962c23e1889cca as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca 2024-11-19T12:53:15,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca, entries=150, sequenceid=146, filesize=11.9 K 2024-11-19T12:53:15,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/37795a227d0b493ba22e1adb25f3cc9f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f 2024-11-19T12:53:15,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f, entries=150, sequenceid=146, filesize=11.9 K 2024-11-19T12:53:15,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 61dc078310d7baf1f63658f1d6fdfff1 in 927ms, sequenceid=146, compaction requested=true 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:15,533 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:15,533 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:15,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:15,535 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:15,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=99.5 K 2024-11-19T12:53:15,535 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/4410bb57284b47999ff884880c152010, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.6 K 2024-11-19T12:53:15,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656] 2024-11-19T12:53:15,535 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4410bb57284b47999ff884880c152010, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:15,537 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91922b159c06498ca06bd104603bd62a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:15,538 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e58699b1e9941bf904f42411aac2539, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732020792399 2024-11-19T12:53:15,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2807054bafa4666acac75a105b8ea19, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732020792399 2024-11-19T12:53:15,538 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a925377a14461fab9eabf74cdc3656, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:15,539 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 913a99f393414922b3962c23e1889cca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:15,546 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:15,547 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/75c5b4145b0241759102f9e50df6981f is 50, key is test_row_0/B:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,551 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:15,560 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111912f1f59ece674148919393fe75450e6f_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:15,562 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111912f1f59ece674148919393fe75450e6f_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:15,562 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912f1f59ece674148919393fe75450e6f_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:15,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742198_1374 (size=12459) 2024-11-19T12:53:15,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742199_1375 (size=4469) 2024-11-19T12:53:15,568 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#321 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:15,568 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/9b566f2595704eb0bbc3af74d7f82adb is 175, key is test_row_0/A:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,574 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/75c5b4145b0241759102f9e50df6981f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/75c5b4145b0241759102f9e50df6981f 2024-11-19T12:53:15,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742200_1376 (size=31413) 2024-11-19T12:53:15,579 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into 75c5b4145b0241759102f9e50df6981f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:15,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:15,579 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=13, startTime=1732020795533; duration=0sec 2024-11-19T12:53:15,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:15,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:15,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:15,580 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:15,580 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:15,580 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:15,580 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/cf6ee12e56984e4d9c9a8caf017bbcf7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=35.6 K 2024-11-19T12:53:15,581 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6ee12e56984e4d9c9a8caf017bbcf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020791651 2024-11-19T12:53:15,581 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6088354fbd4a27abf5666727f5420b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732020792399 2024-11-19T12:53:15,581 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 37795a227d0b493ba22e1adb25f3cc9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:15,587 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:15,587 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/0df34db627f74de3b768d89e85ab01d0 is 50, key is test_row_0/C:col10/1732020794355/Put/seqid=0 2024-11-19T12:53:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742201_1377 (size=12459) 2024-11-19T12:53:15,594 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/0df34db627f74de3b768d89e85ab01d0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/0df34db627f74de3b768d89e85ab01d0 2024-11-19T12:53:15,600 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into 0df34db627f74de3b768d89e85ab01d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:15,600 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:15,600 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=13, startTime=1732020795533; duration=0sec 2024-11-19T12:53:15,600 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:15,600 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:15,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:15,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:15,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b37e66f278654f41b5a6294caf8501a6_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020794731/Put/seqid=0 2024-11-19T12:53:15,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742202_1378 (size=17284) 2024-11-19T12:53:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020855948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020855956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:15,981 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/9b566f2595704eb0bbc3af74d7f82adb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb 2024-11-19T12:53:15,984 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into 9b566f2595704eb0bbc3af74d7f82adb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:15,984 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:15,984 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=13, startTime=1732020795533; duration=0sec 2024-11-19T12:53:15,984 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:15,984 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020856061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020856071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020856282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020856291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:16,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b37e66f278654f41b5a6294caf8501a6_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b37e66f278654f41b5a6294caf8501a6_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:16,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/47a9c028c9b645209a876e2069a2f782, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:16,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/47a9c028c9b645209a876e2069a2f782 is 175, key is test_row_0/A:col10/1732020794731/Put/seqid=0 2024-11-19T12:53:16,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742203_1379 (size=48389) 2024-11-19T12:53:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T12:53:16,442 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:16,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T12:53:16,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:16,444 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:16,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:16,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020856601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020856601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:16,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/47a9c028c9b645209a876e2069a2f782 2024-11-19T12:53:16,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3bfe4d226c854912bd2ef86f21542998 is 50, key is test_row_0/B:col10/1732020794731/Put/seqid=0 2024-11-19T12:53:16,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742204_1380 (size=12151) 2024-11-19T12:53:16,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3bfe4d226c854912bd2ef86f21542998 2024-11-19T12:53:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:16,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 is 50, key is test_row_0/C:col10/1732020794731/Put/seqid=0 2024-11-19T12:53:16,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742205_1381 (size=12151) 2024-11-19T12:53:16,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:16,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:17,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:17,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020857111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020857111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:17,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 2024-11-19T12:53:17,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/47a9c028c9b645209a876e2069a2f782 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782 2024-11-19T12:53:17,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782, entries=250, sequenceid=169, filesize=47.3 K 2024-11-19T12:53:17,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3bfe4d226c854912bd2ef86f21542998 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998 2024-11-19T12:53:17,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T12:53:17,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:17,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 2024-11-19T12:53:17,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:17,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:17,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:17,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T12:53:17,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 61dc078310d7baf1f63658f1d6fdfff1 in 1296ms, sequenceid=169, compaction requested=false 2024-11-19T12:53:17,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:17,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T12:53:17,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:17,361 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190445f7aa21d145e8b0aa3baa8e8bc067_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020795955/Put/seqid=0 2024-11-19T12:53:17,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742206_1382 (size=12304) 2024-11-19T12:53:17,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:17,424 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190445f7aa21d145e8b0aa3baa8e8bc067_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190445f7aa21d145e8b0aa3baa8e8bc067_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:17,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0d51244475fe44349dc241f56077fb42, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0d51244475fe44349dc241f56077fb42 is 175, key is test_row_0/A:col10/1732020795955/Put/seqid=0 2024-11-19T12:53:17,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742207_1383 (size=31105) 2024-11-19T12:53:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:17,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:17,844 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0d51244475fe44349dc241f56077fb42 2024-11-19T12:53:17,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/a4e21e27761a40bf84a063cf7cffa53b is 50, key is test_row_0/B:col10/1732020795955/Put/seqid=0 2024-11-19T12:53:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742208_1384 (size=12151) 2024-11-19T12:53:17,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020857846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:17,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020797816, max timestamp=1732020797856Num uniques 859; 2024-11-19T12:53:17,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:53:17,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020857879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:17,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020857892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:17,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020857971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020858003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020858011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020858131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020858131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,142 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,144 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020858191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020858222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020858222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,260 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/a4e21e27761a40bf84a063cf7cffa53b 2024-11-19T12:53:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc is 50, key is test_row_0/C:col10/1732020795955/Put/seqid=0 2024-11-19T12:53:18,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742209_1385 (size=12151) 2024-11-19T12:53:18,273 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc 2024-11-19T12:53:18,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0d51244475fe44349dc241f56077fb42 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42 2024-11-19T12:53:18,281 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42, entries=150, sequenceid=181, filesize=30.4 K 2024-11-19T12:53:18,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/a4e21e27761a40bf84a063cf7cffa53b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b 2024-11-19T12:53:18,285 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T12:53:18,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc 2024-11-19T12:53:18,290 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T12:53:18,291 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 61dc078310d7baf1f63658f1d6fdfff1 in 929ms, sequenceid=181, compaction requested=true 2024-11-19T12:53:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T12:53:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T12:53:18,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T12:53:18,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-19T12:53:18,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8520 sec 2024-11-19T12:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:18,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:18,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:18,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119433c901cdc354a3ebf64c2b6fea44975_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:18,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020858556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742210_1386 (size=14794) 2024-11-19T12:53:18,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,575 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020858569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119433c901cdc354a3ebf64c2b6fea44975_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119433c901cdc354a3ebf64c2b6fea44975_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:18,580 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/33d0609a752d4c78bb2e60af5f0b1525, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:18,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/33d0609a752d4c78bb2e60af5f0b1525 is 175, key is test_row_0/A:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020858570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T12:53:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,592 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:18,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T12:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:18,597 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:18,598 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:18,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742211_1387 (size=39749) 2024-11-19T12:53:18,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/33d0609a752d4c78bb2e60af5f0b1525 2024-11-19T12:53:18,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/771b054c8caf4ea3bae3249f882836a8 is 50, key is test_row_0/B:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:18,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742212_1388 (size=12151) 2024-11-19T12:53:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020858682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020858682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:18,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020858701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:18,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:18,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:18,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:18,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020858891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:18,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:18,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020858901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:18,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020858921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/771b054c8caf4ea3bae3249f882836a8 2024-11-19T12:53:19,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/89cb3c4cea5348b79dd6e82fe7414da5 is 50, key is test_row_0/C:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742213_1389 (size=12151) 2024-11-19T12:53:19,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/89cb3c4cea5348b79dd6e82fe7414da5 2024-11-19T12:53:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/33d0609a752d4c78bb2e60af5f0b1525 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525 2024-11-19T12:53:19,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525, entries=200, sequenceid=208, filesize=38.8 K 2024-11-19T12:53:19,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/771b054c8caf4ea3bae3249f882836a8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8 2024-11-19T12:53:19,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T12:53:19,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/89cb3c4cea5348b79dd6e82fe7414da5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5 2024-11-19T12:53:19,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T12:53:19,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for 61dc078310d7baf1f63658f1d6fdfff1 in 546ms, sequenceid=208, compaction requested=true 2024-11-19T12:53:19,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:19,067 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:19,067 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:19,068 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:19,068 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150656 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:19,068 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:19,068 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:19,068 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,068 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,068 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/75c5b4145b0241759102f9e50df6981f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=47.8 K 2024-11-19T12:53:19,068 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=147.1 K 2024-11-19T12:53:19,068 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,068 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525] 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c5b4145b0241759102f9e50df6981f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b566f2595704eb0bbc3af74d7f82adb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfe4d226c854912bd2ef86f21542998, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732020794731 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a9c028c9b645209a876e2069a2f782, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732020794697 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4e21e27761a40bf84a063cf7cffa53b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020795927 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d51244475fe44349dc241f56077fb42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020795927 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771b054c8caf4ea3bae3249f882836a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797878 2024-11-19T12:53:19,069 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d0609a752d4c78bb2e60af5f0b1525, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797816 2024-11-19T12:53:19,079 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#332 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:19,079 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/cdc12f94521d440981b2e6e1a283d9a3 is 50, key is test_row_0/B:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:19,080 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,082 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119efbab20d6df24ead89475e0e345cab31_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,085 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119efbab20d6df24ead89475e0e345cab31_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,085 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119efbab20d6df24ead89475e0e345cab31_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742214_1390 (size=12595) 2024-11-19T12:53:19,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742215_1391 (size=4469) 2024-11-19T12:53:19,096 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#333 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:19,097 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/59f0b367d7c64907b3366ef829c85e56 is 175, key is test_row_0/A:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:19,103 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/cdc12f94521d440981b2e6e1a283d9a3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/cdc12f94521d440981b2e6e1a283d9a3 2024-11-19T12:53:19,117 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into cdc12f94521d440981b2e6e1a283d9a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:19,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:19,117 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=12, startTime=1732020799067; duration=0sec 2024-11-19T12:53:19,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:19,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:19,117 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:19,121 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:19,121 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:19,121 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,122 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/0df34db627f74de3b768d89e85ab01d0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=47.8 K 2024-11-19T12:53:19,122 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df34db627f74de3b768d89e85ab01d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732020794355 2024-11-19T12:53:19,122 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3fd8f0fe3b4af99f5e0c13a05ee984, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732020794731 2024-11-19T12:53:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742216_1392 (size=31549) 2024-11-19T12:53:19,126 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8f8d6b01464023a7bc0a9fe1ebe7fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020795927 2024-11-19T12:53:19,130 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cb3c4cea5348b79dd6e82fe7414da5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797878 2024-11-19T12:53:19,157 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:19,158 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/1d9b22fd04e34511a9a8c6beecdfa1a8 is 50, key is test_row_0/C:col10/1732020798519/Put/seqid=0 2024-11-19T12:53:19,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742217_1393 (size=12595) 2024-11-19T12:53:19,183 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/1d9b22fd04e34511a9a8c6beecdfa1a8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/1d9b22fd04e34511a9a8c6beecdfa1a8 2024-11-19T12:53:19,192 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into 1d9b22fd04e34511a9a8c6beecdfa1a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:19,192 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:19,192 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=12, startTime=1732020799067; duration=0sec 2024-11-19T12:53:19,192 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:19,192 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:19,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:19,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111932c9bae2195b4fd58f96e890f9926539_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020799202/Put/seqid=0 2024-11-19T12:53:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:19,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742218_1394 (size=14794) 2024-11-19T12:53:19,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:19,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111932c9bae2195b4fd58f96e890f9926539_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932c9bae2195b4fd58f96e890f9926539_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:19,227 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a662f09cc51c4bdf9ca94aaeebcbc667, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a662f09cc51c4bdf9ca94aaeebcbc667 is 175, key is test_row_0/A:col10/1732020799202/Put/seqid=0 2024-11-19T12:53:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742219_1395 (size=39749) 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020799274, max timestamp=1732020799292Num uniques 859; 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020799274, max timestamp=1732020799292Num uniques 859; 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020799274, max timestamp=1732020799292Num uniques 859; 2024-11-19T12:53:19,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:19,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:19,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020859352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020859373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020859374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020859481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020859501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020859501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:19,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,530 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/59f0b367d7c64907b3366ef829c85e56 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56 2024-11-19T12:53:19,543 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into 59f0b367d7c64907b3366ef829c85e56(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:19,543 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:19,543 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=12, startTime=1732020799067; duration=0sec 2024-11-19T12:53:19,543 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:19,543 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:19,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a662f09cc51c4bdf9ca94aaeebcbc667 2024-11-19T12:53:19,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/574b6d1bb50b43fa85d029bd1c626b1d is 50, key is test_row_0/B:col10/1732020799202/Put/seqid=0 2024-11-19T12:53:19,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:19,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742220_1396 (size=12151) 2024-11-19T12:53:19,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/574b6d1bb50b43fa85d029bd1c626b1d 2024-11-19T12:53:19,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020859701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/004e19729e344ea4a31ff8cda166b7bd is 50, key is test_row_0/C:col10/1732020799202/Put/seqid=0 2024-11-19T12:53:19,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742221_1397 (size=12151) 2024-11-19T12:53:19,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/004e19729e344ea4a31ff8cda166b7bd 2024-11-19T12:53:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020859722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020859722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:19,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/a662f09cc51c4bdf9ca94aaeebcbc667 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667 2024-11-19T12:53:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667, entries=200, sequenceid=221, filesize=38.8 K 2024-11-19T12:53:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/574b6d1bb50b43fa85d029bd1c626b1d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d 2024-11-19T12:53:19,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d, entries=150, sequenceid=221, filesize=11.9 K 2024-11-19T12:53:19,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/004e19729e344ea4a31ff8cda166b7bd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd 2024-11-19T12:53:19,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd, entries=150, sequenceid=221, filesize=11.9 K 2024-11-19T12:53:19,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 61dc078310d7baf1f63658f1d6fdfff1 in 561ms, sequenceid=221, compaction requested=false 2024-11-19T12:53:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:19,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T12:53:19,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:19,829 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:19,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923cc4370d13c4d77a7f60e1ea24f5652_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020799350/Put/seqid=0 2024-11-19T12:53:19,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742222_1398 (size=12304) 2024-11-19T12:53:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:19,848 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923cc4370d13c4d77a7f60e1ea24f5652_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cc4370d13c4d77a7f60e1ea24f5652_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/41041e2bc3da4af489f82f56a7c43f9a, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/41041e2bc3da4af489f82f56a7c43f9a is 175, key is test_row_0/A:col10/1732020799350/Put/seqid=0 2024-11-19T12:53:19,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742223_1399 (size=31105) 2024-11-19T12:53:19,856 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/41041e2bc3da4af489f82f56a7c43f9a 2024-11-19T12:53:19,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/ce2a07330fab46cfa9a9abf560b4d152 is 50, key is test_row_0/B:col10/1732020799350/Put/seqid=0 2024-11-19T12:53:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742224_1400 (size=12151) 2024-11-19T12:53:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:20,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020860057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020860069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020860070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020860151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,162 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020860161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,164 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020860182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020860191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020860191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,285 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/ce2a07330fab46cfa9a9abf560b4d152 2024-11-19T12:53:20,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b56d9564f275405a9bcc0e891fee34fd is 50, key is test_row_0/C:col10/1732020799350/Put/seqid=0 2024-11-19T12:53:20,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742225_1401 (size=12151) 2024-11-19T12:53:20,300 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b56d9564f275405a9bcc0e891fee34fd 2024-11-19T12:53:20,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/41041e2bc3da4af489f82f56a7c43f9a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a 2024-11-19T12:53:20,309 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a, entries=150, sequenceid=256, filesize=30.4 K 2024-11-19T12:53:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/ce2a07330fab46cfa9a9abf560b4d152 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152 2024-11-19T12:53:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,316 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152, entries=150, sequenceid=256, filesize=11.9 K 2024-11-19T12:53:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b56d9564f275405a9bcc0e891fee34fd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd 2024-11-19T12:53:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,322 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd, entries=150, sequenceid=256, filesize=11.9 K 2024-11-19T12:53:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,323 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 61dc078310d7baf1f63658f1d6fdfff1 in 494ms, sequenceid=256, compaction requested=true 2024-11-19T12:53:20,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:20,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T12:53:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T12:53:20,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7260 sec 2024-11-19T12:53:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7350 sec 2024-11-19T12:53:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:53:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:20,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119597da486556c487a95bb6f784004b28a_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742226_1402 (size=12454) 2024-11-19T12:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119597da486556c487a95bb6f784004b28a_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119597da486556c487a95bb6f784004b28a_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ef1091e093d84c42b312498a16c98e02, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ef1091e093d84c42b312498a16c98e02 is 175, key is test_row_0/A:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742227_1403 (size=31251) 2024-11-19T12:53:20,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ef1091e093d84c42b312498a16c98e02 2024-11-19T12:53:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/77da997d6bff4e1e940ee029ee09c3fe is 50, key is test_row_0/B:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742228_1404 (size=12297) 2024-11-19T12:53:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:20,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020860537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020860546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020860549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020860652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020860674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020860677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T12:53:20,752 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:20,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T12:53:20,753 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T12:53:20,754 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:20,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T12:53:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020860871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/77da997d6bff4e1e940ee029ee09c3fe 2024-11-19T12:53:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020860891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020860891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:20,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T12:53:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:20,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:20,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/eb86d6680b5c4a24a9a476d03c23cef7 is 50, key is test_row_0/C:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:20,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742229_1405 (size=9857) 2024-11-19T12:53:20,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/eb86d6680b5c4a24a9a476d03c23cef7 2024-11-19T12:53:20,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/ef1091e093d84c42b312498a16c98e02 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02 2024-11-19T12:53:20,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02, entries=150, sequenceid=268, filesize=30.5 K 2024-11-19T12:53:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/77da997d6bff4e1e940ee029ee09c3fe as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe 2024-11-19T12:53:20,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T12:53:20,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/eb86d6680b5c4a24a9a476d03c23cef7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7 2024-11-19T12:53:20,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7, entries=100, sequenceid=268, filesize=9.6 K 2024-11-19T12:53:20,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 61dc078310d7baf1f63658f1d6fdfff1 in 545ms, sequenceid=268, compaction requested=true 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:20,968 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:20,968 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:20,969 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:20,969 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:20,969 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:20,969 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:20,969 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,970 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/cdc12f94521d440981b2e6e1a283d9a3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=48.0 K 2024-11-19T12:53:20,970 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,970 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=130.5 K 2024-11-19T12:53:20,970 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:20,970 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02] 2024-11-19T12:53:20,970 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59f0b367d7c64907b3366ef829c85e56, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797878 2024-11-19T12:53:20,971 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc12f94521d440981b2e6e1a283d9a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797878 2024-11-19T12:53:20,971 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a662f09cc51c4bdf9ca94aaeebcbc667, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732020798554 2024-11-19T12:53:20,971 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 574b6d1bb50b43fa85d029bd1c626b1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732020798567 2024-11-19T12:53:20,973 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2a07330fab46cfa9a9abf560b4d152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732020799322 2024-11-19T12:53:20,973 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41041e2bc3da4af489f82f56a7c43f9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732020799322 2024-11-19T12:53:20,974 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 77da997d6bff4e1e940ee029ee09c3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020800065 2024-11-19T12:53:20,974 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1091e093d84c42b312498a16c98e02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020800065 2024-11-19T12:53:20,991 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:21,009 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:21,010 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8d469fb1d8104ff68c48020cafa521a1 is 50, key is test_row_0/B:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:21,027 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191cc463a6203c4da2bc8359fa0d2d0ba8_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:21,030 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191cc463a6203c4da2bc8359fa0d2d0ba8_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:21,030 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191cc463a6203c4da2bc8359fa0d2d0ba8_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:21,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T12:53:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:21,060 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:53:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T12:53:21,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742230_1406 (size=12831) 2024-11-19T12:53:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742231_1407 (size=4469) 2024-11-19T12:53:21,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119507ffe891b144c4e997d4b0c4143451f_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020800527/Put/seqid=0 2024-11-19T12:53:21,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742232_1408 (size=12454) 2024-11-19T12:53:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:21,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:21,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020861231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020861233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020861241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020861351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020861351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020861361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T12:53:21,492 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/8d469fb1d8104ff68c48020cafa521a1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8d469fb1d8104ff68c48020cafa521a1 2024-11-19T12:53:21,498 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into 8d469fb1d8104ff68c48020cafa521a1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:21,498 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:21,498 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=12, startTime=1732020800968; duration=0sec 2024-11-19T12:53:21,499 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:21,499 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:21,499 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:21,499 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46754 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:21,500 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:21,500 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:21,500 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/1d9b22fd04e34511a9a8c6beecdfa1a8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=45.7 K 2024-11-19T12:53:21,500 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9b22fd04e34511a9a8c6beecdfa1a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732020797878 2024-11-19T12:53:21,500 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 004e19729e344ea4a31ff8cda166b7bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732020798567 2024-11-19T12:53:21,504 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b56d9564f275405a9bcc0e891fee34fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732020799322 2024-11-19T12:53:21,504 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting eb86d6680b5c4a24a9a476d03c23cef7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020800068 2024-11-19T12:53:21,511 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:21,511 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/3ff2574de4fe408a9733736209a452b6 is 50, key is test_row_0/C:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:21,514 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#344 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:21,514 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/c2d6abc6d01640b2af7be9839f165b16 is 175, key is test_row_0/A:col10/1732020800420/Put/seqid=0 2024-11-19T12:53:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742233_1409 (size=12831) 2024-11-19T12:53:21,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742234_1410 (size=31892) 2024-11-19T12:53:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:21,544 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119507ffe891b144c4e997d4b0c4143451f_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119507ffe891b144c4e997d4b0c4143451f_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:21,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/210fd41a6a2f466cb793891ddee4578f, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/210fd41a6a2f466cb793891ddee4578f is 175, key is test_row_0/A:col10/1732020800527/Put/seqid=0 2024-11-19T12:53:21,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742235_1411 (size=31255) 2024-11-19T12:53:21,557 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/210fd41a6a2f466cb793891ddee4578f 2024-11-19T12:53:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/41d9d58fc5f44c2b89429ac2e344f8d2 is 50, key is test_row_0/B:col10/1732020800527/Put/seqid=0 2024-11-19T12:53:21,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742236_1412 (size=12301) 2024-11-19T12:53:21,582 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/41d9d58fc5f44c2b89429ac2e344f8d2 2024-11-19T12:53:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020861571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020861571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 is 50, key is test_row_0/C:col10/1732020800527/Put/seqid=0 2024-11-19T12:53:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020861582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742237_1413 (size=12301) 2024-11-19T12:53:21,608 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 2024-11-19T12:53:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/210fd41a6a2f466cb793891ddee4578f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f 2024-11-19T12:53:21,624 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f, entries=150, sequenceid=288, filesize=30.5 K 2024-11-19T12:53:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/41d9d58fc5f44c2b89429ac2e344f8d2 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2 2024-11-19T12:53:21,631 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T12:53:21,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 2024-11-19T12:53:21,636 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T12:53:21,637 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 61dc078310d7baf1f63658f1d6fdfff1 in 577ms, sequenceid=288, compaction requested=false 2024-11-19T12:53:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T12:53:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T12:53:21,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T12:53:21,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 884 msec 2024-11-19T12:53:21,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 889 msec 2024-11-19T12:53:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T12:53:21,892 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:21,892 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T12:53:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:21,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:21,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:21,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:21,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:21,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198673c03bec5943b8ad7328c1756938e4_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:21,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742238_1414 (size=14994) 2024-11-19T12:53:21,919 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/3ff2574de4fe408a9733736209a452b6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/3ff2574de4fe408a9733736209a452b6 2024-11-19T12:53:21,923 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/c2d6abc6d01640b2af7be9839f165b16 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16 2024-11-19T12:53:21,924 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into 3ff2574de4fe408a9733736209a452b6(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:21,924 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:21,924 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=12, startTime=1732020800968; duration=0sec 2024-11-19T12:53:21,924 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:21,924 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:21,927 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into c2d6abc6d01640b2af7be9839f165b16(size=31.1 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:21,927 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:21,927 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=12, startTime=1732020800968; duration=0sec 2024-11-19T12:53:21,927 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:21,927 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020861950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020861963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:21,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020861964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:22,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T12:53:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020862072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020862072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020862082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T12:53:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:22,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020862283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020862291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020862301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,316 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198673c03bec5943b8ad7328c1756938e4_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198673c03bec5943b8ad7328c1756938e4_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:22,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e195a9adff1e4836a44da7c0b76dbf12, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e195a9adff1e4836a44da7c0b76dbf12 is 175, key is test_row_0/A:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742239_1415 (size=39949) 2024-11-19T12:53:22,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e195a9adff1e4836a44da7c0b76dbf12 2024-11-19T12:53:22,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/750ab499e3864926af5efbe685f73a8c is 50, key is test_row_0/B:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T12:53:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742240_1416 (size=12301) 2024-11-19T12:53:22,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/750ab499e3864926af5efbe685f73a8c 2024-11-19T12:53:22,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/06531a721b59457fa30abb5a4c833a7d is 50, key is test_row_0/C:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742241_1417 (size=12301) 2024-11-19T12:53:22,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/06531a721b59457fa30abb5a4c833a7d 2024-11-19T12:53:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e195a9adff1e4836a44da7c0b76dbf12 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12 2024-11-19T12:53:22,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12, entries=200, sequenceid=302, filesize=39.0 K 2024-11-19T12:53:22,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/750ab499e3864926af5efbe685f73a8c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c 2024-11-19T12:53:22,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c, entries=150, sequenceid=302, filesize=12.0 K 2024-11-19T12:53:22,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/06531a721b59457fa30abb5a4c833a7d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d 2024-11-19T12:53:22,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d, entries=150, sequenceid=302, filesize=12.0 K 2024-11-19T12:53:22,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 61dc078310d7baf1f63658f1d6fdfff1 in 518ms, sequenceid=302, compaction requested=true 2024-11-19T12:53:22,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:22,417 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:22,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:22,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:22,417 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:22,419 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103096 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:22,419 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:22,419 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,419 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:22,419 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=100.7 K 2024-11-19T12:53:22,419 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:22,419 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,419 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,419 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12] 2024-11-19T12:53:22,419 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8d469fb1d8104ff68c48020cafa521a1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=36.6 K 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2d6abc6d01640b2af7be9839f165b16, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020799370 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d469fb1d8104ff68c48020cafa521a1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020799370 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 210fd41a6a2f466cb793891ddee4578f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732020800527 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 41d9d58fc5f44c2b89429ac2e344f8d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732020800527 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e195a9adff1e4836a44da7c0b76dbf12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:22,422 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 750ab499e3864926af5efbe685f73a8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:22,428 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,429 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:22,429 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc is 50, key is test_row_0/B:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,435 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119860a8dcce8944076b818d6e2d3fb1b7b_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,437 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119860a8dcce8944076b818d6e2d3fb1b7b_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,437 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119860a8dcce8944076b818d6e2d3fb1b7b_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742242_1418 (size=12983) 2024-11-19T12:53:22,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742243_1419 (size=4469) 2024-11-19T12:53:22,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T12:53:22,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,509 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c112e856f3c43ee8accfdc3c2fe638d_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020801945/Put/seqid=0 2024-11-19T12:53:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:22,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742244_1420 (size=12454) 2024-11-19T12:53:22,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:22,528 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c112e856f3c43ee8accfdc3c2fe638d_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c112e856f3c43ee8accfdc3c2fe638d_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/06ab4f69d11d4de7b156f4f9de7e6f19, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/06ab4f69d11d4de7b156f4f9de7e6f19 is 175, key is test_row_0/A:col10/1732020801945/Put/seqid=0 2024-11-19T12:53:22,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742245_1421 (size=31255) 2024-11-19T12:53:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020862639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020862640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020862641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020862761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020862761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020862761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,848 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc 2024-11-19T12:53:22,851 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:22,851 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/5b6d47d147d344239a4353fcc64f493e is 175, key is test_row_0/A:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,852 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into 6e797fdaa60c4c7bbc6c1a4c4fe7a3cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:22,852 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:22,852 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=13, startTime=1732020802417; duration=0sec 2024-11-19T12:53:22,852 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:22,852 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:22,852 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:22,853 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:22,853 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:22,853 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:22,853 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/3ff2574de4fe408a9733736209a452b6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=36.6 K 2024-11-19T12:53:22,854 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff2574de4fe408a9733736209a452b6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732020799370 2024-11-19T12:53:22,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742246_1422 (size=31937) 2024-11-19T12:53:22,855 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f56e0bbc7e456cbdb42bb33c6bd8f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732020800527 2024-11-19T12:53:22,855 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 06531a721b59457fa30abb5a4c833a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:22,862 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/5b6d47d147d344239a4353fcc64f493e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e 2024-11-19T12:53:22,865 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:22,866 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/dfccf169653a48769d5500555150746a is 50, key is test_row_0/C:col10/1732020801239/Put/seqid=0 2024-11-19T12:53:22,867 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into 5b6d47d147d344239a4353fcc64f493e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:22,867 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:22,868 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=13, startTime=1732020802417; duration=0sec 2024-11-19T12:53:22,868 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:22,868 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:22,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742247_1423 (size=12983) 2024-11-19T12:53:22,891 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/dfccf169653a48769d5500555150746a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/dfccf169653a48769d5500555150746a 2024-11-19T12:53:22,898 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into dfccf169653a48769d5500555150746a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:22,898 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:22,898 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=13, startTime=1732020802419; duration=0sec 2024-11-19T12:53:22,898 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:22,898 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:22,938 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/06ab4f69d11d4de7b156f4f9de7e6f19 2024-11-19T12:53:22,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/fdcf974adc8e436f8caf712a69e5acfc is 50, key is test_row_0/B:col10/1732020801945/Put/seqid=0 2024-11-19T12:53:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742248_1424 (size=12301) 2024-11-19T12:53:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020862971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020862971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020862971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020863281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020863292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020863292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,352 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/fdcf974adc8e436f8caf712a69e5acfc 2024-11-19T12:53:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/88206dba53364534805d25f1d6fc3066 is 50, key is test_row_0/C:col10/1732020801945/Put/seqid=0 2024-11-19T12:53:23,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742249_1425 (size=12301) 2024-11-19T12:53:23,395 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/88206dba53364534805d25f1d6fc3066 2024-11-19T12:53:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/06ab4f69d11d4de7b156f4f9de7e6f19 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19 2024-11-19T12:53:23,403 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19, entries=150, sequenceid=323, filesize=30.5 K 2024-11-19T12:53:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/fdcf974adc8e436f8caf712a69e5acfc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc 2024-11-19T12:53:23,408 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc, entries=150, sequenceid=323, filesize=12.0 K 2024-11-19T12:53:23,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/88206dba53364534805d25f1d6fc3066 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066 2024-11-19T12:53:23,412 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066, entries=150, sequenceid=323, filesize=12.0 K 2024-11-19T12:53:23,413 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 61dc078310d7baf1f63658f1d6fdfff1 in 904ms, sequenceid=323, compaction requested=false 2024-11-19T12:53:23,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:23,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:23,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T12:53:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T12:53:23,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T12:53:23,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-19T12:53:23,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5260 sec 2024-11-19T12:53:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:23,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:23,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:23,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afbf4cf5717943969228bd38715b5d6b_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:23,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742250_1426 (size=14994) 2024-11-19T12:53:23,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020863852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afbf4cf5717943969228bd38715b5d6b_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbf4cf5717943969228bd38715b5d6b_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020863863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e1f2dc9b6bb3439f8b4d6ad29703ac23, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:23,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 is 175, key is test_row_0/A:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:23,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742251_1427 (size=39949) 2024-11-19T12:53:23,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020863865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020863972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020863972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020863981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T12:53:24,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:24,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T12:53:24,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:24,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:24,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47582 deadline: 1732020864181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,191 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020864191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020864192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47584 deadline: 1732020864201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020864201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,204 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 61dc078310d7baf1f63658f1d6fdfff1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 2024-11-19T12:53:24,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/abc164c6d3754db89cafb5f801a84318 is 50, key is test_row_0/B:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:24,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742252_1428 (size=12301) 2024-11-19T12:53:24,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:24,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020864511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020864512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020864512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:24,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:24,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/abc164c6d3754db89cafb5f801a84318 2024-11-19T12:53:24,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b720a0c144b448b0a9edbd9dbe5307de is 50, key is test_row_0/C:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:24,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742253_1429 (size=12301) 2024-11-19T12:53:24,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:24,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:25,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47568 deadline: 1732020865031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47508 deadline: 1732020865031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1732020865031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:53:25,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b720a0c144b448b0a9edbd9dbe5307de 2024-11-19T12:53:25,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 2024-11-19T12:53:25,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23, entries=200, sequenceid=340, filesize=39.0 K 2024-11-19T12:53:25,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/abc164c6d3754db89cafb5f801a84318 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318 2024-11-19T12:53:25,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. as already flushing 2024-11-19T12:53:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:25,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T12:53:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:25,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/b720a0c144b448b0a9edbd9dbe5307de as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de 2024-11-19T12:53:25,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T12:53:25,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 61dc078310d7baf1f63658f1d6fdfff1 in 1315ms, sequenceid=340, compaction requested=true 2024-11-19T12:53:25,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:25,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:25,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:25,122 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:25,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:25,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:25,122 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:25,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61dc078310d7baf1f63658f1d6fdfff1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:25,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:25,122 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:25,122 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/B is initiating minor compaction (all files) 2024-11-19T12:53:25,122 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/B in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,123 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=36.7 K 2024-11-19T12:53:25,124 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e797fdaa60c4c7bbc6c1a4c4fe7a3cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:25,124 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:25,124 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting fdcf974adc8e436f8caf712a69e5acfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732020801945 2024-11-19T12:53:25,124 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/A is initiating minor compaction (all files) 2024-11-19T12:53:25,124 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/A in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,124 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=100.7 K 2024-11-19T12:53:25,125 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting abc164c6d3754db89cafb5f801a84318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732020802623 2024-11-19T12:53:25,125 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,125 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23] 2024-11-19T12:53:25,125 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6d47d147d344239a4353fcc64f493e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:25,125 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ab4f69d11d4de7b156f4f9de7e6f19, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732020801945 2024-11-19T12:53:25,125 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f2dc9b6bb3439f8b4d6ad29703ac23, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732020802623 2024-11-19T12:53:25,131 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:25,131 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#B#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:25,132 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3214490de8a54c8a8a8688aa402c2532 is 50, key is test_row_0/B:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:25,133 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411190819497c53d4418b865e35d43849b83b_61dc078310d7baf1f63658f1d6fdfff1 store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:25,135 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411190819497c53d4418b865e35d43849b83b_61dc078310d7baf1f63658f1d6fdfff1, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:25,135 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190819497c53d4418b865e35d43849b83b_61dc078310d7baf1f63658f1d6fdfff1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742254_1430 (size=13085) 2024-11-19T12:53:25,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742255_1431 (size=4469) 2024-11-19T12:53:25,178 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#A#compaction#363 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:25,179 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/6405c2c3570749d1a3966ac30cfd1e0c is 175, key is test_row_0/A:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:25,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742256_1432 (size=32039) 2024-11-19T12:53:25,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,269 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:25,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac81c571569948f6bef021594794dfbb_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_0/A:col10/1732020803863/Put/seqid=0 2024-11-19T12:53:25,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742257_1433 (size=12454) 2024-11-19T12:53:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:25,361 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac81c571569948f6bef021594794dfbb_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac81c571569948f6bef021594794dfbb_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0112fb2d328743fab1a0e7380ba3cedb, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0112fb2d328743fab1a0e7380ba3cedb is 175, key is test_row_0/A:col10/1732020803863/Put/seqid=0 2024-11-19T12:53:25,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742258_1434 (size=31255) 2024-11-19T12:53:25,569 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/3214490de8a54c8a8a8688aa402c2532 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3214490de8a54c8a8a8688aa402c2532 2024-11-19T12:53:25,578 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/B of 61dc078310d7baf1f63658f1d6fdfff1 into 3214490de8a54c8a8a8688aa402c2532(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:25,578 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:25,578 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/B, priority=13, startTime=1732020805122; duration=0sec 2024-11-19T12:53:25,578 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:25,578 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:B 2024-11-19T12:53:25,578 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:25,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:25,579 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 61dc078310d7baf1f63658f1d6fdfff1/C is initiating minor compaction (all files) 2024-11-19T12:53:25,579 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61dc078310d7baf1f63658f1d6fdfff1/C in TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,579 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/dfccf169653a48769d5500555150746a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp, totalSize=36.7 K 2024-11-19T12:53:25,580 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting dfccf169653a48769d5500555150746a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732020801195 2024-11-19T12:53:25,580 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 88206dba53364534805d25f1d6fc3066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732020801945 2024-11-19T12:53:25,580 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b720a0c144b448b0a9edbd9dbe5307de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732020802623 2024-11-19T12:53:25,602 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61dc078310d7baf1f63658f1d6fdfff1#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:25,602 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/fac278cedb9b44248d0e1793abfb49f6 is 50, key is test_row_0/C:col10/1732020803804/Put/seqid=0 2024-11-19T12:53:25,604 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/6405c2c3570749d1a3966ac30cfd1e0c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/6405c2c3570749d1a3966ac30cfd1e0c 2024-11-19T12:53:25,609 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/A of 61dc078310d7baf1f63658f1d6fdfff1 into 6405c2c3570749d1a3966ac30cfd1e0c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:25,609 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:25,609 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/A, priority=13, startTime=1732020805121; duration=0sec 2024-11-19T12:53:25,609 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:25,609 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:A 2024-11-19T12:53:25,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742259_1435 (size=13085) 2024-11-19T12:53:25,651 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/fac278cedb9b44248d0e1793abfb49f6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/fac278cedb9b44248d0e1793abfb49f6 2024-11-19T12:53:25,656 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61dc078310d7baf1f63658f1d6fdfff1/C of 61dc078310d7baf1f63658f1d6fdfff1 into fac278cedb9b44248d0e1793abfb49f6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:25,657 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:25,657 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., storeName=61dc078310d7baf1f63658f1d6fdfff1/C, priority=13, startTime=1732020805122; duration=0sec 2024-11-19T12:53:25,657 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:25,657 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61dc078310d7baf1f63658f1d6fdfff1:C 2024-11-19T12:53:25,818 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0112fb2d328743fab1a0e7380ba3cedb 2024-11-19T12:53:25,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/d4296a8ba8dd43c0b627e7d445ce715e is 50, key is test_row_0/B:col10/1732020803863/Put/seqid=0 2024-11-19T12:53:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742260_1436 (size=12301) 2024-11-19T12:53:25,927 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/d4296a8ba8dd43c0b627e7d445ce715e 2024-11-19T12:53:25,936 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-19T12:53:25,936 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:25,936 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,936 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:25,942 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-19T12:53:25,942 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:25,942 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,942 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:25,943 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-19T12:53:25,943 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:25,943 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,943 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e956b87234c44265ad90a4417709c56d is 50, key is test_row_0/C:col10/1732020803863/Put/seqid=0 2024-11-19T12:53:25,947 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-19T12:53:25,947 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:25,947 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,947 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:25,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742261_1437 (size=12301) 2024-11-19T12:53:25,951 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e956b87234c44265ad90a4417709c56d 2024-11-19T12:53:25,952 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-19T12:53:25,953 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:25,953 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,953 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:25,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/0112fb2d328743fab1a0e7380ba3cedb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0112fb2d328743fab1a0e7380ba3cedb 2024-11-19T12:53:25,961 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0112fb2d328743fab1a0e7380ba3cedb, entries=150, sequenceid=358, filesize=30.5 K 2024-11-19T12:53:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/d4296a8ba8dd43c0b627e7d445ce715e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/d4296a8ba8dd43c0b627e7d445ce715e 2024-11-19T12:53:25,966 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/d4296a8ba8dd43c0b627e7d445ce715e, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T12:53:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/e956b87234c44265ad90a4417709c56d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e956b87234c44265ad90a4417709c56d 2024-11-19T12:53:25,971 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e956b87234c44265ad90a4417709c56d, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T12:53:25,972 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 61dc078310d7baf1f63658f1d6fdfff1 in 703ms, sequenceid=358, compaction requested=false 2024-11-19T12:53:25,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 61dc078310d7baf1f63658f1d6fdfff1: 2024-11-19T12:53:25,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:25,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T12:53:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T12:53:25,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T12:53:25,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-19T12:53:25,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.9340 sec 2024-11-19T12:53:26,053 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-19T12:53:26,053 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-19T12:53:26,053 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:26,053 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:26,053 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,053 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,054 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,054 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:26,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:26,054 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-11-19T12:53:26,055 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:26,055 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,055 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:26,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T12:53:26,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:32,958 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:53:34,206 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-19T12:53:34,206 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:34,206 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,206 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:34,224 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-19T12:53:34,224 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:34,225 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,225 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:34,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:53:34,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T12:53:34,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-19T12:53:34,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-19T12:53:34,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1011 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3028 rows 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3024 rows 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1015 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3045 rows 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3045 rows 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1007 2024-11-19T12:53:34,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3021 rows 2024-11-19T12:53:34,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:53:34,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:53:34,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:34,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:34,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:53:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:34,235 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020814235"}]},"ts":"1732020814235"} 2024-11-19T12:53:34,237 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:53:34,237 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:53:34,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:53:34,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, UNASSIGN}] 2024-11-19T12:53:34,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, UNASSIGN 2024-11-19T12:53:34,241 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:34,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, UNASSIGN because future has completed 2024-11-19T12:53:34,242 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:53:34,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:34,394 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:34,394 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:53:34,394 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 61dc078310d7baf1f63658f1d6fdfff1, disabling compactions & flushes 2024-11-19T12:53:34,394 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. after waiting 0 ms 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:34,395 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 61dc078310d7baf1f63658f1d6fdfff1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=A 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=B 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61dc078310d7baf1f63658f1d6fdfff1, store=C 2024-11-19T12:53:34,395 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:34,405 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197ab8cddfa3774185aeca701357846912_61dc078310d7baf1f63658f1d6fdfff1 is 50, key is test_row_1/A:col10/1732020814203/Put/seqid=0 2024-11-19T12:53:34,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742262_1438 (size=9914) 2024-11-19T12:53:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:34,808 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:34,812 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197ab8cddfa3774185aeca701357846912_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197ab8cddfa3774185aeca701357846912_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:34,813 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/cbf5f1fc329149a98d9632ee98464dce, store: [table=TestAcidGuarantees family=A region=61dc078310d7baf1f63658f1d6fdfff1] 2024-11-19T12:53:34,813 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/cbf5f1fc329149a98d9632ee98464dce is 175, key is test_row_1/A:col10/1732020814203/Put/seqid=0 2024-11-19T12:53:34,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742263_1439 (size=22561) 2024-11-19T12:53:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:35,217 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/cbf5f1fc329149a98d9632ee98464dce 2024-11-19T12:53:35,223 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/64372300c31b4e11863c0e97b39d4231 is 50, key is test_row_1/B:col10/1732020814203/Put/seqid=0 2024-11-19T12:53:35,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742264_1440 (size=9857) 2024-11-19T12:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:35,626 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/64372300c31b4e11863c0e97b39d4231 2024-11-19T12:53:35,644 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/83ca0c13187d423ba6f30e203c9e28f0 is 50, key is test_row_1/C:col10/1732020814203/Put/seqid=0 2024-11-19T12:53:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742265_1441 (size=9857) 2024-11-19T12:53:36,048 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/83ca0c13187d423ba6f30e203c9e28f0 2024-11-19T12:53:36,053 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/A/cbf5f1fc329149a98d9632ee98464dce as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/cbf5f1fc329149a98d9632ee98464dce 2024-11-19T12:53:36,057 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/cbf5f1fc329149a98d9632ee98464dce, entries=100, sequenceid=369, filesize=22.0 K 2024-11-19T12:53:36,058 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/B/64372300c31b4e11863c0e97b39d4231 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/64372300c31b4e11863c0e97b39d4231 2024-11-19T12:53:36,061 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/64372300c31b4e11863c0e97b39d4231, entries=100, sequenceid=369, filesize=9.6 K 2024-11-19T12:53:36,062 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/.tmp/C/83ca0c13187d423ba6f30e203c9e28f0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/83ca0c13187d423ba6f30e203c9e28f0 2024-11-19T12:53:36,065 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/83ca0c13187d423ba6f30e203c9e28f0, entries=100, sequenceid=369, filesize=9.6 K 2024-11-19T12:53:36,066 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 61dc078310d7baf1f63658f1d6fdfff1 in 1671ms, sequenceid=369, compaction requested=true 2024-11-19T12:53:36,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23] to archive 2024-11-19T12:53:36,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:36,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/449aef31ae054f7b8f008ab4aa663309 2024-11-19T12:53:36,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/248e6e561b9345f2bde464f493f43543 2024-11-19T12:53:36,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/f1d555ed6d1a4a979ba1a6e457415349 2024-11-19T12:53:36,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ed71091db78e4d66abfd95358cd162ae 2024-11-19T12:53:36,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b45707d02b5c4a069137f987b21cc6f3 2024-11-19T12:53:36,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/bc0e043c79174d7080b6d8062b3e4da5 2024-11-19T12:53:36,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/91922b159c06498ca06bd104603bd62a 2024-11-19T12:53:36,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/b2807054bafa4666acac75a105b8ea19 2024-11-19T12:53:36,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a2a925377a14461fab9eabf74cdc3656 2024-11-19T12:53:36,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/9b566f2595704eb0bbc3af74d7f82adb 2024-11-19T12:53:36,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/47a9c028c9b645209a876e2069a2f782 2024-11-19T12:53:36,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0d51244475fe44349dc241f56077fb42 2024-11-19T12:53:36,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/33d0609a752d4c78bb2e60af5f0b1525 2024-11-19T12:53:36,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/59f0b367d7c64907b3366ef829c85e56 2024-11-19T12:53:36,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/a662f09cc51c4bdf9ca94aaeebcbc667 2024-11-19T12:53:36,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/41041e2bc3da4af489f82f56a7c43f9a 2024-11-19T12:53:36,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/c2d6abc6d01640b2af7be9839f165b16 2024-11-19T12:53:36,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/ef1091e093d84c42b312498a16c98e02 2024-11-19T12:53:36,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/210fd41a6a2f466cb793891ddee4578f 2024-11-19T12:53:36,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e195a9adff1e4836a44da7c0b76dbf12 2024-11-19T12:53:36,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/5b6d47d147d344239a4353fcc64f493e 2024-11-19T12:53:36,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/06ab4f69d11d4de7b156f4f9de7e6f19 2024-11-19T12:53:36,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/e1f2dc9b6bb3439f8b4d6ad29703ac23 2024-11-19T12:53:36,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c0062c572c7949d3ba8d849057cb185a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/4410bb57284b47999ff884880c152010, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/75c5b4145b0241759102f9e50df6981f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/cdc12f94521d440981b2e6e1a283d9a3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8d469fb1d8104ff68c48020cafa521a1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318] to archive 2024-11-19T12:53:36,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:36,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8adabcd6a5f54672a421029a2607eb41 2024-11-19T12:53:36,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/aa23780b991044c28c6b253f1cc05ee3 2024-11-19T12:53:36,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c0062c572c7949d3ba8d849057cb185a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c0062c572c7949d3ba8d849057cb185a 2024-11-19T12:53:36,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3b76e12d534b48378fba100ddb7a4787 2024-11-19T12:53:36,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/7aa9cb3fd0554c2c90e01697484b7812 2024-11-19T12:53:36,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/4410bb57284b47999ff884880c152010 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/4410bb57284b47999ff884880c152010 2024-11-19T12:53:36,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/c412bbadf9de4a17b23b1801df4886ce 2024-11-19T12:53:36,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8e58699b1e9941bf904f42411aac2539 2024-11-19T12:53:36,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/75c5b4145b0241759102f9e50df6981f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/75c5b4145b0241759102f9e50df6981f 2024-11-19T12:53:36,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/913a99f393414922b3962c23e1889cca 2024-11-19T12:53:36,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3bfe4d226c854912bd2ef86f21542998 2024-11-19T12:53:36,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/a4e21e27761a40bf84a063cf7cffa53b 2024-11-19T12:53:36,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/cdc12f94521d440981b2e6e1a283d9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/cdc12f94521d440981b2e6e1a283d9a3 2024-11-19T12:53:36,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/771b054c8caf4ea3bae3249f882836a8 2024-11-19T12:53:36,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/574b6d1bb50b43fa85d029bd1c626b1d 2024-11-19T12:53:36,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/ce2a07330fab46cfa9a9abf560b4d152 2024-11-19T12:53:36,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8d469fb1d8104ff68c48020cafa521a1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/8d469fb1d8104ff68c48020cafa521a1 2024-11-19T12:53:36,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/77da997d6bff4e1e940ee029ee09c3fe 2024-11-19T12:53:36,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/41d9d58fc5f44c2b89429ac2e344f8d2 2024-11-19T12:53:36,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/6e797fdaa60c4c7bbc6c1a4c4fe7a3cc 2024-11-19T12:53:36,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/750ab499e3864926af5efbe685f73a8c 2024-11-19T12:53:36,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/fdcf974adc8e436f8caf712a69e5acfc 2024-11-19T12:53:36,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/abc164c6d3754db89cafb5f801a84318 2024-11-19T12:53:36,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/886c5d7f8efd4c34850c3fba21e9091e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/cf6ee12e56984e4d9c9a8caf017bbcf7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/0df34db627f74de3b768d89e85ab01d0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/1d9b22fd04e34511a9a8c6beecdfa1a8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/3ff2574de4fe408a9733736209a452b6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/dfccf169653a48769d5500555150746a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de] to archive 2024-11-19T12:53:36,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:53:36,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e7083807fd38440eb415ffbb4ee14725 2024-11-19T12:53:36,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5a4b184d212940cbaffef61bacb30da7 2024-11-19T12:53:36,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/886c5d7f8efd4c34850c3fba21e9091e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/886c5d7f8efd4c34850c3fba21e9091e 2024-11-19T12:53:36,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f78647a406da47588eab4970bdf01086 2024-11-19T12:53:36,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/5045b9e17c064a1fa2f3ff46b5caf2d7 2024-11-19T12:53:36,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/cf6ee12e56984e4d9c9a8caf017bbcf7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/cf6ee12e56984e4d9c9a8caf017bbcf7 2024-11-19T12:53:36,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/2be46b1047774dff9f0e9fc52f7a8549 2024-11-19T12:53:36,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/ad6088354fbd4a27abf5666727f5420b 2024-11-19T12:53:36,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/0df34db627f74de3b768d89e85ab01d0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/0df34db627f74de3b768d89e85ab01d0 2024-11-19T12:53:36,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/37795a227d0b493ba22e1adb25f3cc9f 2024-11-19T12:53:36,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/aa3fd8f0fe3b4af99f5e0c13a05ee984 2024-11-19T12:53:36,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/4d8f8d6b01464023a7bc0a9fe1ebe7fc 2024-11-19T12:53:36,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/1d9b22fd04e34511a9a8c6beecdfa1a8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/1d9b22fd04e34511a9a8c6beecdfa1a8 2024-11-19T12:53:36,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/89cb3c4cea5348b79dd6e82fe7414da5 2024-11-19T12:53:36,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/004e19729e344ea4a31ff8cda166b7bd 2024-11-19T12:53:36,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b56d9564f275405a9bcc0e891fee34fd 2024-11-19T12:53:36,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/3ff2574de4fe408a9733736209a452b6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/3ff2574de4fe408a9733736209a452b6 2024-11-19T12:53:36,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/eb86d6680b5c4a24a9a476d03c23cef7 2024-11-19T12:53:36,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/f0f56e0bbc7e456cbdb42bb33c6bd8f1 2024-11-19T12:53:36,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/dfccf169653a48769d5500555150746a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/dfccf169653a48769d5500555150746a 2024-11-19T12:53:36,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/06531a721b59457fa30abb5a4c833a7d 2024-11-19T12:53:36,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/88206dba53364534805d25f1d6fc3066 2024-11-19T12:53:36,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/b720a0c144b448b0a9edbd9dbe5307de 2024-11-19T12:53:36,135 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits/372.seqid, newMaxSeqId=372, maxSeqId=4 2024-11-19T12:53:36,135 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. 2024-11-19T12:53:36,136 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 61dc078310d7baf1f63658f1d6fdfff1: Waiting for close lock at 1732020814394Running coprocessor pre-close hooks at 1732020814394Disabling compacts and flushes for region at 1732020814394Disabling writes for close at 1732020814395 (+1 ms)Obtaining lock to block concurrent updates at 1732020814395Preparing flush snapshotting stores in 61dc078310d7baf1f63658f1d6fdfff1 at 1732020814395Finished memstore snapshotting TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732020814395Flushing stores of TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1. at 1732020814396 (+1 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/A: creating writer at 1732020814396Flushing mob file 61dc078310d7baf1f63658f1d6fdfff1/A: appending metadata at 1732020814404 (+8 ms)Flushing mob file 61dc078310d7baf1f63658f1d6fdfff1/A: closing flushed file at 1732020814404Flushing 61dc078310d7baf1f63658f1d6fdfff1/A: appending metadata at 1732020814813 (+409 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/A: closing flushed file at 1732020814813Flushing 61dc078310d7baf1f63658f1d6fdfff1/B: creating writer at 1732020815220 (+407 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/B: appending metadata at 1732020815222 (+2 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/B: closing flushed file at 1732020815222Flushing 61dc078310d7baf1f63658f1d6fdfff1/C: creating writer at 1732020815631 (+409 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/C: appending metadata at 1732020815643 (+12 ms)Flushing 61dc078310d7baf1f63658f1d6fdfff1/C: closing flushed file at 1732020815643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e57c59d: reopening flushed file at 1732020816052 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15b984c: reopening flushed file at 1732020816057 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c85c6da: reopening flushed file at 1732020816061 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 61dc078310d7baf1f63658f1d6fdfff1 in 1671ms, sequenceid=369, compaction requested=true at 1732020816066 (+5 ms)Writing region close event to WAL at 1732020816131 (+65 ms)Running coprocessor post-close hooks at 1732020816135 (+4 ms)Closed at 1732020816135 2024-11-19T12:53:36,138 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,139 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=61dc078310d7baf1f63658f1d6fdfff1, regionState=CLOSED 2024-11-19T12:53:36,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:36,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T12:53:36,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 61dc078310d7baf1f63658f1d6fdfff1, server=aba5a916dfea,35349,1732020694570 in 1.9000 sec 2024-11-19T12:53:36,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-19T12:53:36,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61dc078310d7baf1f63658f1d6fdfff1, UNASSIGN in 1.9120 sec 2024-11-19T12:53:36,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T12:53:36,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-19T12:53:36,157 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020816157"}]},"ts":"1732020816157"} 2024-11-19T12:53:36,159 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:53:36,159 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:53:36,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9300 sec 2024-11-19T12:53:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T12:53:36,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:36,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:53:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,385 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T12:53:36,385 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,391 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits] 2024-11-19T12:53:36,395 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0112fb2d328743fab1a0e7380ba3cedb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/0112fb2d328743fab1a0e7380ba3cedb 2024-11-19T12:53:36,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/6405c2c3570749d1a3966ac30cfd1e0c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/6405c2c3570749d1a3966ac30cfd1e0c 2024-11-19T12:53:36,397 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/cbf5f1fc329149a98d9632ee98464dce to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/A/cbf5f1fc329149a98d9632ee98464dce 2024-11-19T12:53:36,399 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3214490de8a54c8a8a8688aa402c2532 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/3214490de8a54c8a8a8688aa402c2532 2024-11-19T12:53:36,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/64372300c31b4e11863c0e97b39d4231 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/64372300c31b4e11863c0e97b39d4231 2024-11-19T12:53:36,401 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/d4296a8ba8dd43c0b627e7d445ce715e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/B/d4296a8ba8dd43c0b627e7d445ce715e 2024-11-19T12:53:36,403 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/83ca0c13187d423ba6f30e203c9e28f0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/83ca0c13187d423ba6f30e203c9e28f0 2024-11-19T12:53:36,404 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e956b87234c44265ad90a4417709c56d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/e956b87234c44265ad90a4417709c56d 2024-11-19T12:53:36,405 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/fac278cedb9b44248d0e1793abfb49f6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/C/fac278cedb9b44248d0e1793abfb49f6 2024-11-19T12:53:36,407 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits/372.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1/recovered.edits/372.seqid 2024-11-19T12:53:36,408 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,408 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:53:36,408 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:53:36,409 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T12:53:36,411 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190445f7aa21d145e8b0aa3baa8e8bc067_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190445f7aa21d145e8b0aa3baa8e8bc067_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,412 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ae6c7bcae3f4f999f427a9b759d5225_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ae6c7bcae3f4f999f427a9b759d5225_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d1f866b9b324250b3233aeb9a9dc8a7_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d1f866b9b324250b3233aeb9a9dc8a7_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119111884f835f94347bf452e9136b7401b_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119111884f835f94347bf452e9136b7401b_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,414 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cf92b6b221c4fd8b53a936d3a8fe6ba_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cf92b6b221c4fd8b53a936d3a8fe6ba_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,415 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cc4370d13c4d77a7f60e1ea24f5652_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111923cc4370d13c4d77a7f60e1ea24f5652_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,415 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111924c3e0c6280c4035bf972dc4f49cdeb4_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111924c3e0c6280c4035bf972dc4f49cdeb4_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,416 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932c9bae2195b4fd58f96e890f9926539_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932c9bae2195b4fd58f96e890f9926539_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119433c901cdc354a3ebf64c2b6fea44975_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119433c901cdc354a3ebf64c2b6fea44975_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,418 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119507ffe891b144c4e997d4b0c4143451f_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119507ffe891b144c4e997d4b0c4143451f_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,419 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119597da486556c487a95bb6f784004b28a_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119597da486556c487a95bb6f784004b28a_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,419 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197ab8cddfa3774185aeca701357846912_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197ab8cddfa3774185aeca701357846912_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,420 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198673c03bec5943b8ad7328c1756938e4_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198673c03bec5943b8ad7328c1756938e4_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c112e856f3c43ee8accfdc3c2fe638d_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199c112e856f3c43ee8accfdc3c2fe638d_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac81c571569948f6bef021594794dfbb_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac81c571569948f6bef021594794dfbb_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,422 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbf4cf5717943969228bd38715b5d6b_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbf4cf5717943969228bd38715b5d6b_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,423 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b37e66f278654f41b5a6294caf8501a6_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b37e66f278654f41b5a6294caf8501a6_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,423 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6dddf23164f4a199ff776123b6180ee_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d6dddf23164f4a199ff776123b6180ee_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,424 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119edd7405d1d894d2f8587c51c9e639a85_61dc078310d7baf1f63658f1d6fdfff1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119edd7405d1d894d2f8587c51c9e639a85_61dc078310d7baf1f63658f1d6fdfff1 2024-11-19T12:53:36,424 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:53:36,426 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,427 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:53:36,429 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:53:36,430 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,430 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:53:36,430 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020816430"}]},"ts":"9223372036854775807"} 2024-11-19T12:53:36,432 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:53:36,432 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 61dc078310d7baf1f63658f1d6fdfff1, NAME => 'TestAcidGuarantees,,1732020782706.61dc078310d7baf1f63658f1d6fdfff1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:53:36,432 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:53:36,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020816432"}]},"ts":"9223372036854775807"} 2024-11-19T12:53:36,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:53:36,434 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-19T12:53:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T12:53:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:53:36,492 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:36,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 252), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=529 (was 424) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5426 (was 5616) 2024-11-19T12:53:36,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=529, ProcessCount=11, AvailableMemoryMB=5425 2024-11-19T12:53:36,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:53:36,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:53:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:53:36,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:53:36,512 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:36,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-19T12:53:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:36,513 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:53:36,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742266_1442 (size=1061) 2024-11-19T12:53:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:36,921 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:53:36,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742267_1443 (size=53) 2024-11-19T12:53:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1fb4fabd22f8f9ba7eb58c23e43c4dba, disabling compactions & flushes 2024-11-19T12:53:37,328 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. after waiting 0 ms 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,328 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: Waiting for close lock at 1732020817328Disabling compacts and flushes for region at 1732020817328Disabling writes for close at 1732020817328Writing region close event to WAL at 1732020817328Closed at 1732020817328 2024-11-19T12:53:37,330 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:53:37,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020817330"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020817330"}]},"ts":"1732020817330"} 2024-11-19T12:53:37,332 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:53:37,333 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:53:37,333 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020817333"}]},"ts":"1732020817333"} 2024-11-19T12:53:37,335 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:53:37,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, ASSIGN}] 2024-11-19T12:53:37,336 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, ASSIGN 2024-11-19T12:53:37,337 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:53:37,488 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1fb4fabd22f8f9ba7eb58c23e43c4dba, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:37,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, ASSIGN because future has completed 2024-11-19T12:53:37,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:53:37,645 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,645 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:53:37,646 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,646 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:53:37,646 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,646 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,647 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,648 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:37,649 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fb4fabd22f8f9ba7eb58c23e43c4dba columnFamilyName A 2024-11-19T12:53:37,649 DEBUG [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:37,649 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(327): Store=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:37,649 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,650 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:37,650 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fb4fabd22f8f9ba7eb58c23e43c4dba columnFamilyName B 2024-11-19T12:53:37,650 DEBUG [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:37,651 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(327): Store=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:37,651 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,652 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:53:37,652 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fb4fabd22f8f9ba7eb58c23e43c4dba columnFamilyName C 2024-11-19T12:53:37,652 DEBUG [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:53:37,653 INFO [StoreOpener-1fb4fabd22f8f9ba7eb58c23e43c4dba-1 {}] regionserver.HStore(327): Store=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:53:37,653 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,653 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,654 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,654 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,655 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,655 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,656 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:53:37,657 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,658 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:53:37,659 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 1fb4fabd22f8f9ba7eb58c23e43c4dba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72335744, jitterRate=0.07788658142089844}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:53:37,659 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:37,660 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: Running coprocessor pre-open hook at 1732020817646Writing region info on filesystem at 1732020817646Initializing all the Stores at 1732020817647 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020817647Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020817647Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020817647Cleaning up temporary data from old regions at 1732020817655 (+8 ms)Running coprocessor post-open hooks at 1732020817659 (+4 ms)Region opened successfully at 1732020817660 (+1 ms) 2024-11-19T12:53:37,661 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., pid=116, masterSystemTime=1732020817642 2024-11-19T12:53:37,663 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,663 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:37,664 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1fb4fabd22f8f9ba7eb58c23e43c4dba, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:37,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:53:37,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T12:53:37,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 in 176 msec 2024-11-19T12:53:37,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-19T12:53:37,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, ASSIGN in 333 msec 2024-11-19T12:53:37,672 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:53:37,672 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020817672"}]},"ts":"1732020817672"} 2024-11-19T12:53:37,674 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:53:37,675 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:53:37,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-19T12:53:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-19T12:53:38,662 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:38,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d124c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db0532a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,668 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:53:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:53:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8419f8b 2024-11-19T12:53:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:53:38,670 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:53:38,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,672 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7828ba70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e500b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17703bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666d9d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,679 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76541712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22306292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433806b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fea80fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bdc389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718cfd84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658a0893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@252cd499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c40c98e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd6927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516deb53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712bcafb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,698 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6551cbfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128648b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:53:38,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:53:38,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7f3be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:53:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:38,702 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:53:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b03ce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:53:38,703 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,703 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,703 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,703 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,703 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,704 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,704 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,705 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,706 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:38,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T12:53:38,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,707 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:53:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T12:53:38,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,709 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:38,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:38,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:38,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:53:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:53:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:53:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2] 2024-11-19T12:53:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:38,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:38,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732020818734, max timestamp=1732020818744Num uniques 859; 2024-11-19T12:53:38,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:38,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ef0fa5d47a5040fdb6e825e3e398780e is 50, key is test_row_0/A:col10/1732020818718/Put/seqid=0 2024-11-19T12:53:38,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020818734, max timestamp=1732020818744Num uniques 859; 2024-11-19T12:53:38,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020818734, max timestamp=1732020818744Num uniques 859; 2024-11-19T12:53:38,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:38,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742268_1444 (size=14341) 2024-11-19T12:53:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020878768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020878776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020878776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020878777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020878779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T12:53:38,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T12:53:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:38,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:38,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020878891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020878892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020878892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020878901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020878901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:38,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:38,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T12:53:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T12:53:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020879102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020879102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020879111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020879122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020879124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ef0fa5d47a5040fdb6e825e3e398780e 2024-11-19T12:53:39,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T12:53:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:39,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/66a22284c7524981ae27ebcd2608930c is 50, key is test_row_0/B:col10/1732020818718/Put/seqid=0 2024-11-19T12:53:39,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742269_1445 (size=12001) 2024-11-19T12:53:39,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/66a22284c7524981ae27ebcd2608930c 2024-11-19T12:53:39,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/40d08e2957f04aeca09c5c8a403548c8 is 50, key is test_row_0/C:col10/1732020818718/Put/seqid=0 2024-11-19T12:53:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742270_1446 (size=12001) 2024-11-19T12:53:39,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/40d08e2957f04aeca09c5c8a403548c8 2024-11-19T12:53:39,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ef0fa5d47a5040fdb6e825e3e398780e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e 2024-11-19T12:53:39,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e, entries=200, sequenceid=11, filesize=14.0 K 2024-11-19T12:53:39,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/66a22284c7524981ae27ebcd2608930c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c 2024-11-19T12:53:39,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T12:53:39,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/40d08e2957f04aeca09c5c8a403548c8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8 2024-11-19T12:53:39,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T12:53:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 579ms, sequenceid=11, compaction requested=false 2024-11-19T12:53:39,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:39,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T12:53:39,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,326 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T12:53:39,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c50519523e6d474284184cdf67530e30 is 50, key is test_row_0/A:col10/1732020818775/Put/seqid=0 2024-11-19T12:53:39,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742271_1447 (size=12001) 2024-11-19T12:53:39,351 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c50519523e6d474284184cdf67530e30 2024-11-19T12:53:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ebc493da1de4c11b7d153b29d74ec30 is 50, key is test_row_0/B:col10/1732020818775/Put/seqid=0 2024-11-19T12:53:39,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742272_1448 (size=12001) 2024-11-19T12:53:39,383 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ebc493da1de4c11b7d153b29d74ec30 2024-11-19T12:53:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ab6e7f7a06244125a8a4fcf2b12317b8 is 50, key is test_row_0/C:col10/1732020818775/Put/seqid=0 2024-11-19T12:53:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:39,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:39,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742273_1449 (size=12001) 2024-11-19T12:53:39,428 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ab6e7f7a06244125a8a4fcf2b12317b8 2024-11-19T12:53:39,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c50519523e6d474284184cdf67530e30 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30 2024-11-19T12:53:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020879441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020879441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,448 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T12:53:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020879442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ebc493da1de4c11b7d153b29d74ec30 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30 2024-11-19T12:53:39,453 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T12:53:39,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ab6e7f7a06244125a8a4fcf2b12317b8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8 2024-11-19T12:53:39,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020879444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020879443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,459 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T12:53:39,460 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 134ms, sequenceid=46, compaction requested=false 2024-11-19T12:53:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T12:53:39,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T12:53:39,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 752 msec 2024-11-19T12:53:39,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 759 msec 2024-11-19T12:53:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:39,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:39,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:39,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/1341662044ec429fb4d8a7a5bd8878aa is 50, key is test_row_0/A:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:39,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742274_1450 (size=14341) 2024-11-19T12:53:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020879624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020879633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020819614, max timestamp=1732020819634Num uniques 859; 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020819614, max timestamp=1732020819634Num uniques 859; 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020819614, max timestamp=1732020819634Num uniques 859; 2024-11-19T12:53:39,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:53:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020879704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020879705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020879742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020879742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020879821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020879822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T12:53:39,851 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:39,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T12:53:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:39,853 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:39,854 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:39,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:39,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020879951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020879961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020879961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/1341662044ec429fb4d8a7a5bd8878aa 2024-11-19T12:53:40,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T12:53:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/cdce841b5d064016afd7f77530b80fb7 is 50, key is test_row_0/B:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:40,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742275_1451 (size=12001) 2024-11-19T12:53:40,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020880041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020880041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T12:53:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:40,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020880271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020880271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T12:53:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:40,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020880361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020880361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/cdce841b5d064016afd7f77530b80fb7 2024-11-19T12:53:40,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/2f67b891de6f4089989048b7a21fa77c is 50, key is test_row_0/C:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:40,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742276_1452 (size=12001) 2024-11-19T12:53:40,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/2f67b891de6f4089989048b7a21fa77c 2024-11-19T12:53:40,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/1341662044ec429fb4d8a7a5bd8878aa as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa 2024-11-19T12:53:40,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa, entries=200, sequenceid=57, filesize=14.0 K 2024-11-19T12:53:40,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/cdce841b5d064016afd7f77530b80fb7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7 2024-11-19T12:53:40,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7, entries=150, sequenceid=57, filesize=11.7 K 2024-11-19T12:53:40,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/2f67b891de6f4089989048b7a21fa77c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c 2024-11-19T12:53:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c, entries=150, sequenceid=57, filesize=11.7 K 2024-11-19T12:53:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 891ms, sequenceid=57, compaction requested=true 2024-11-19T12:53:40,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:40,449 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:40,449 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,449 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,450 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.2 K 2024-11-19T12:53:40,450 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=39.7 K 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a22284c7524981ae27ebcd2608930c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732020818714 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef0fa5d47a5040fdb6e825e3e398780e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732020818713 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebc493da1de4c11b7d153b29d74ec30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020818766 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c50519523e6d474284184cdf67530e30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020818766 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting cdce841b5d064016afd7f77530b80fb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819442 2024-11-19T12:53:40,450 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1341662044ec429fb4d8a7a5bd8878aa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819441 2024-11-19T12:53:40,457 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:40,457 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8bb5ec7ff87848f393fbcab1c2a3dd52 is 50, key is test_row_0/A:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:40,460 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:40,460 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/05c27b07687e4ee4bf75505b98907b66 is 50, key is test_row_0/B:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742277_1453 (size=12104) 2024-11-19T12:53:40,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T12:53:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,465 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-19T12:53:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:40,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:40,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742278_1454 (size=12104) 2024-11-19T12:53:40,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ce4ef40c574f429886f0f878e4bd1b95 is 50, key is test_row_0/A:col10/1732020819704/Put/seqid=0 2024-11-19T12:53:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742279_1455 (size=12001) 2024-11-19T12:53:40,477 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/05c27b07687e4ee4bf75505b98907b66 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/05c27b07687e4ee4bf75505b98907b66 2024-11-19T12:53:40,480 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 05c27b07687e4ee4bf75505b98907b66(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:40,480 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:40,480 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020820449; duration=0sec 2024-11-19T12:53:40,480 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:40,480 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:40,480 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:40,483 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:40,483 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:40,483 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:40,483 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.2 K 2024-11-19T12:53:40,483 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d08e2957f04aeca09c5c8a403548c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732020818714 2024-11-19T12:53:40,484 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6e7f7a06244125a8a4fcf2b12317b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732020818766 2024-11-19T12:53:40,484 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f67b891de6f4089989048b7a21fa77c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819442 2024-11-19T12:53:40,499 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:40,500 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/cc9d1b8d16004ab3abcdf10a15d25ced is 50, key is test_row_0/C:col10/1732020819443/Put/seqid=0 2024-11-19T12:53:40,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742280_1456 (size=12104) 2024-11-19T12:53:40,532 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/cc9d1b8d16004ab3abcdf10a15d25ced as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/cc9d1b8d16004ab3abcdf10a15d25ced 2024-11-19T12:53:40,536 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into cc9d1b8d16004ab3abcdf10a15d25ced(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:40,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:40,536 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020820449; duration=0sec 2024-11-19T12:53:40,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:40,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:40,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020880848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020880849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,869 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8bb5ec7ff87848f393fbcab1c2a3dd52 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8bb5ec7ff87848f393fbcab1c2a3dd52 2024-11-19T12:53:40,873 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 8bb5ec7ff87848f393fbcab1c2a3dd52(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:40,873 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:40,873 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020820448; duration=0sec 2024-11-19T12:53:40,874 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:40,874 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:40,876 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ce4ef40c574f429886f0f878e4bd1b95 2024-11-19T12:53:40,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/04f7371db4ac472dae949d66c7041175 is 50, key is test_row_0/B:col10/1732020819704/Put/seqid=0 2024-11-19T12:53:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020880881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020880881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742281_1457 (size=12001) 2024-11-19T12:53:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020880961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020880972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020880972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:40,981 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:41,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020881181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020881191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:41,297 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/04f7371db4ac472dae949d66c7041175 2024-11-19T12:53:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/afb059673f80437e813e4bc00acddcc0 is 50, key is test_row_0/C:col10/1732020819704/Put/seqid=0 2024-11-19T12:53:41,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742282_1458 (size=12001) 2024-11-19T12:53:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020881491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020881511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:41,671 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:53:41,723 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/afb059673f80437e813e4bc00acddcc0 2024-11-19T12:53:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ce4ef40c574f429886f0f878e4bd1b95 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95 2024-11-19T12:53:41,730 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T12:53:41,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/04f7371db4ac472dae949d66c7041175 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175 2024-11-19T12:53:41,733 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T12:53:41,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/afb059673f80437e813e4bc00acddcc0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0 2024-11-19T12:53:41,736 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T12:53:41,737 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1272ms, sequenceid=91, compaction requested=false 2024-11-19T12:53:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:41,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T12:53:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T12:53:41,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T12:53:41,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-19T12:53:41,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8880 sec 2024-11-19T12:53:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:41,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:41,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:41,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/63a3006354a846959746c10b3ce24d88 is 50, key is test_row_0/A:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:41,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742283_1459 (size=12001) 2024-11-19T12:53:41,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020881978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020881980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T12:53:42,001 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:42,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020882001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T12:53:42,007 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T12:53:42,007 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:42,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:42,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020882022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020882101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020882101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T12:53:42,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:42,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020882311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020882312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T12:53:42,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/63a3006354a846959746c10b3ce24d88 2024-11-19T12:53:42,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/c8c2a9af5d28480f9f08e9a4a4126d72 is 50, key is test_row_0/B:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:42,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742284_1460 (size=12001) 2024-11-19T12:53:42,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T12:53:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020882631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:42,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020882631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:42,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/c8c2a9af5d28480f9f08e9a4a4126d72 2024-11-19T12:53:42,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/4f0ef5050e69466dba4f169270403005 is 50, key is test_row_0/C:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:42,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742285_1461 (size=12001) 2024-11-19T12:53:42,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/4f0ef5050e69466dba4f169270403005 2024-11-19T12:53:42,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/63a3006354a846959746c10b3ce24d88 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88 2024-11-19T12:53:42,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:42,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T12:53:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/c8c2a9af5d28480f9f08e9a4a4126d72 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72 2024-11-19T12:53:42,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T12:53:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/4f0ef5050e69466dba4f169270403005 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005 2024-11-19T12:53:42,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005, entries=150, sequenceid=107, filesize=11.7 K 2024-11-19T12:53:42,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 877ms, sequenceid=107, compaction requested=true 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:42,785 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:42,785 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:42,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:42,787 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,787 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,787 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/05c27b07687e4ee4bf75505b98907b66, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.3 K 2024-11-19T12:53:42,787 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8bb5ec7ff87848f393fbcab1c2a3dd52, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.3 K 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c27b07687e4ee4bf75505b98907b66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819442 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb5ec7ff87848f393fbcab1c2a3dd52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819442 2024-11-19T12:53:42,787 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 04f7371db4ac472dae949d66c7041175, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732020819681 2024-11-19T12:53:42,788 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4ef40c574f429886f0f878e4bd1b95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732020819681 2024-11-19T12:53:42,788 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c2a9af5d28480f9f08e9a4a4126d72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:42,788 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a3006354a846959746c10b3ce24d88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:42,796 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:42,797 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/147bdb43e9bf4043aaf94f4d7f9c6afb is 50, key is test_row_0/B:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:42,797 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:42,797 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/fb6b080c3bf8407785958e6c76cf2239 is 50, key is test_row_0/A:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742286_1462 (size=12207) 2024-11-19T12:53:42,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742287_1463 (size=12207) 2024-11-19T12:53:42,810 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/147bdb43e9bf4043aaf94f4d7f9c6afb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/147bdb43e9bf4043aaf94f4d7f9c6afb 2024-11-19T12:53:42,816 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 147bdb43e9bf4043aaf94f4d7f9c6afb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:42,816 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:42,816 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020822785; duration=0sec 2024-11-19T12:53:42,816 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:42,816 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:42,816 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:42,817 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:42,817 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:42,817 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,817 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/cc9d1b8d16004ab3abcdf10a15d25ced, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.3 K 2024-11-19T12:53:42,817 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9d1b8d16004ab3abcdf10a15d25ced, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732020819442 2024-11-19T12:53:42,817 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting afb059673f80437e813e4bc00acddcc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732020819681 2024-11-19T12:53:42,817 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0ef5050e69466dba4f169270403005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:42,822 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:42,823 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/aad06023433044e2a89676a87e4f6ccd is 50, key is test_row_0/C:col10/1732020821905/Put/seqid=0 2024-11-19T12:53:42,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742288_1464 (size=12207) 2024-11-19T12:53:42,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:42,929 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/850be3efd72c40ea920c54619332637e is 50, key is test_row_0/A:col10/1732020821975/Put/seqid=0 2024-11-19T12:53:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742289_1465 (size=12001) 2024-11-19T12:53:42,945 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/850be3efd72c40ea920c54619332637e 2024-11-19T12:53:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d is 50, key is test_row_0/B:col10/1732020821975/Put/seqid=0 2024-11-19T12:53:42,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742290_1466 (size=12001) 2024-11-19T12:53:42,996 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d 2024-11-19T12:53:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/75cb393378e14882b38bb05145132f44 is 50, key is test_row_0/C:col10/1732020821975/Put/seqid=0 2024-11-19T12:53:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:43,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742291_1467 (size=12001) 2024-11-19T12:53:43,032 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/75cb393378e14882b38bb05145132f44 2024-11-19T12:53:43,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/850be3efd72c40ea920c54619332637e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e 2024-11-19T12:53:43,043 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T12:53:43,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d 2024-11-19T12:53:43,051 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T12:53:43,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/75cb393378e14882b38bb05145132f44 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44 2024-11-19T12:53:43,055 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T12:53:43,056 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 127ms, sequenceid=127, compaction requested=false 2024-11-19T12:53:43,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:43,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T12:53:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T12:53:43,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T12:53:43,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-11-19T12:53:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:43,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:43,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8be600bbc63b43efb6a8917e7df11ebb is 50, key is test_row_0/A:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:43,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0570 sec 2024-11-19T12:53:43,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742292_1468 (size=14541) 2024-11-19T12:53:43,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020883094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020883104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020883106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T12:53:43,142 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:43,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T12:53:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:43,144 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:43,144 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:43,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:43,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020883142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020883151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,213 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/fb6b080c3bf8407785958e6c76cf2239 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/fb6b080c3bf8407785958e6c76cf2239 2024-11-19T12:53:43,217 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into fb6b080c3bf8407785958e6c76cf2239(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:43,217 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:43,217 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020822785; duration=0sec 2024-11-19T12:53:43,217 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:43,217 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020883212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020883212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,229 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/aad06023433044e2a89676a87e4f6ccd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/aad06023433044e2a89676a87e4f6ccd 2024-11-19T12:53:43,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020883221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,235 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into aad06023433044e2a89676a87e4f6ccd(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:43,235 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:43,235 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020822785; duration=0sec 2024-11-19T12:53:43,235 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:43,235 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:43,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:43,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020883422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020883433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020883442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:43,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8be600bbc63b43efb6a8917e7df11ebb 2024-11-19T12:53:43,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/b1c8df9633a84daa92ad269eed3783a9 is 50, key is test_row_0/B:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:43,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742293_1469 (size=12151) 2024-11-19T12:53:43,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020883741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020883751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020883762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:43,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/b1c8df9633a84daa92ad269eed3783a9 2024-11-19T12:53:43,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8bd11e3bb1a54852814f1803068805fb is 50, key is test_row_0/C:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:43,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742294_1470 (size=12151) 2024-11-19T12:53:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:43,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8bd11e3bb1a54852814f1803068805fb 2024-11-19T12:53:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8be600bbc63b43efb6a8917e7df11ebb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb 2024-11-19T12:53:43,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb, entries=200, sequenceid=141, filesize=14.2 K 2024-11-19T12:53:43,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/b1c8df9633a84daa92ad269eed3783a9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9 2024-11-19T12:53:43,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9, entries=150, sequenceid=141, filesize=11.9 K 2024-11-19T12:53:43,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8bd11e3bb1a54852814f1803068805fb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb 2024-11-19T12:53:43,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb, entries=150, sequenceid=141, filesize=11.9 K 2024-11-19T12:53:43,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 870ms, sequenceid=141, compaction requested=true 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:43,932 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:43,932 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:43,933 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:43,933 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:43,933 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,933 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/fb6b080c3bf8407785958e6c76cf2239, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=37.8 K 2024-11-19T12:53:43,933 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:43,933 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:43,933 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,933 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/147bdb43e9bf4043aaf94f4d7f9c6afb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.5 K 2024-11-19T12:53:43,933 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6b080c3bf8407785958e6c76cf2239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:43,934 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 147bdb43e9bf4043aaf94f4d7f9c6afb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:43,934 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 850be3efd72c40ea920c54619332637e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732020821970 2024-11-19T12:53:43,934 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec7dfe6b1934ad4ad2d8f3511aa5c3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732020821970 2024-11-19T12:53:43,934 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be600bbc63b43efb6a8917e7df11ebb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:43,934 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c8df9633a84daa92ad269eed3783a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:43,945 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:43,946 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/84bb9befbdd6474f8e4a399ec443783e is 50, key is test_row_0/B:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:43,952 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:43,953 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/2c78a7cc6ed7497d8f583a6df633bfc0 is 50, key is test_row_0/A:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742295_1471 (size=12459) 2024-11-19T12:53:43,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742296_1472 (size=12459) 2024-11-19T12:53:43,973 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/2c78a7cc6ed7497d8f583a6df633bfc0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2c78a7cc6ed7497d8f583a6df633bfc0 2024-11-19T12:53:43,978 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 2c78a7cc6ed7497d8f583a6df633bfc0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:43,978 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:43,978 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020823932; duration=0sec 2024-11-19T12:53:43,978 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:43,979 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:43,979 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:43,979 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:43,979 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:43,980 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:43,980 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/aad06023433044e2a89676a87e4f6ccd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.5 K 2024-11-19T12:53:43,980 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad06023433044e2a89676a87e4f6ccd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732020820836 2024-11-19T12:53:43,980 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75cb393378e14882b38bb05145132f44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732020821970 2024-11-19T12:53:43,980 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd11e3bb1a54852814f1803068805fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:43,999 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:44,000 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/6233694980ed49afa772df09397ac40e is 50, key is test_row_0/C:col10/1732020823048/Put/seqid=0 2024-11-19T12:53:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742297_1473 (size=12459) 2024-11-19T12:53:44,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:44,060 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:44,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:44,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/449a0bbfdafb456c83544a8116d90c10 is 50, key is test_row_0/A:col10/1732020823090/Put/seqid=0 2024-11-19T12:53:44,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742298_1474 (size=12151) 2024-11-19T12:53:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:44,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:44,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020884209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020884213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020884261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020884271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:44,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020884282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020884321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020884331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,372 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/84bb9befbdd6474f8e4a399ec443783e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/84bb9befbdd6474f8e4a399ec443783e 2024-11-19T12:53:44,376 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 84bb9befbdd6474f8e4a399ec443783e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:44,376 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:44,376 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020823932; duration=0sec 2024-11-19T12:53:44,376 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:44,376 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:44,416 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/6233694980ed49afa772df09397ac40e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/6233694980ed49afa772df09397ac40e 2024-11-19T12:53:44,420 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 6233694980ed49afa772df09397ac40e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:44,420 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:44,420 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020823932; duration=0sec 2024-11-19T12:53:44,420 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:44,420 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:44,507 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/449a0bbfdafb456c83544a8116d90c10 2024-11-19T12:53:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b is 50, key is test_row_0/B:col10/1732020823090/Put/seqid=0 2024-11-19T12:53:44,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742299_1475 (size=12151) 2024-11-19T12:53:44,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020884541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020884553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020884851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020884861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:44,917 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b 2024-11-19T12:53:44,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d1e70ac1b2e6408aa5858c1028f54288 is 50, key is test_row_0/C:col10/1732020823090/Put/seqid=0 2024-11-19T12:53:44,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742300_1476 (size=12151) 2024-11-19T12:53:45,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020885281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,288 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:45,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020885291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,294 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020885301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,310 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,332 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d1e70ac1b2e6408aa5858c1028f54288 2024-11-19T12:53:45,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/449a0bbfdafb456c83544a8116d90c10 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10 2024-11-19T12:53:45,341 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T12:53:45,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b 2024-11-19T12:53:45,345 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T12:53:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d1e70ac1b2e6408aa5858c1028f54288 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288 2024-11-19T12:53:45,348 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288, entries=150, sequenceid=162, filesize=11.9 K 2024-11-19T12:53:45,349 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1289ms, sequenceid=162, compaction requested=false 2024-11-19T12:53:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T12:53:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T12:53:45,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T12:53:45,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-19T12:53:45,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2090 sec 2024-11-19T12:53:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:45,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:45,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/74f57e23777444fcaf85ef47e852ea69 is 50, key is test_row_0/A:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742301_1477 (size=14541) 2024-11-19T12:53:45,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/74f57e23777444fcaf85ef47e852ea69 2024-11-19T12:53:45,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ea97a19c40a48cb9410a863f950bdfc is 50, key is test_row_0/B:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742302_1478 (size=12151) 2024-11-19T12:53:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020885475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020885475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020885591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020885602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ea97a19c40a48cb9410a863f950bdfc 2024-11-19T12:53:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020885801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/44f2c127f0914b868b66b24aed904686 is 50, key is test_row_0/C:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020885813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742303_1479 (size=12151) 2024-11-19T12:53:45,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/44f2c127f0914b868b66b24aed904686 2024-11-19T12:53:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/74f57e23777444fcaf85ef47e852ea69 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69 2024-11-19T12:53:45,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69, entries=200, sequenceid=179, filesize=14.2 K 2024-11-19T12:53:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3ea97a19c40a48cb9410a863f950bdfc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc 2024-11-19T12:53:45,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T12:53:45,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/44f2c127f0914b868b66b24aed904686 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686 2024-11-19T12:53:45,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686, entries=150, sequenceid=179, filesize=11.9 K 2024-11-19T12:53:45,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 466ms, sequenceid=179, compaction requested=true 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:45,841 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:45,841 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:45,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:45,841 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:45,841 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:45,841 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:45,842 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2c78a7cc6ed7497d8f583a6df633bfc0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=38.2 K 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:45,842 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:45,842 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/84bb9befbdd6474f8e4a399ec443783e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.9 K 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c78a7cc6ed7497d8f583a6df633bfc0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 84bb9befbdd6474f8e4a399ec443783e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449a0bbfdafb456c83544a8116d90c10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732020823090 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f57e23777444fcaf85ef47e852ea69, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:45,842 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a1e235eceb4ee1b7dbcc8cde16cf1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732020823090 2024-11-19T12:53:45,843 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea97a19c40a48cb9410a863f950bdfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:45,852 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:45,852 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:45,853 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8016d40adce742fc9736e4c23a65f37a is 50, key is test_row_0/A:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,853 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/65d32696c7b340e398dee65bdba15a75 is 50, key is test_row_0/B:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742305_1481 (size=12561) 2024-11-19T12:53:45,872 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/65d32696c7b340e398dee65bdba15a75 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/65d32696c7b340e398dee65bdba15a75 2024-11-19T12:53:45,880 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 65d32696c7b340e398dee65bdba15a75(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:45,880 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:45,880 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020825841; duration=0sec 2024-11-19T12:53:45,880 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:45,880 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:45,880 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:45,881 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:45,881 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:45,881 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:45,881 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/6233694980ed49afa772df09397ac40e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=35.9 K 2024-11-19T12:53:45,881 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6233694980ed49afa772df09397ac40e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732020823045 2024-11-19T12:53:45,881 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e70ac1b2e6408aa5858c1028f54288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732020823090 2024-11-19T12:53:45,882 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f2c127f0914b868b66b24aed904686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:45,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742304_1480 (size=12561) 2024-11-19T12:53:45,890 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:45,891 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/851b0cd87b9049d6a38b250f82f59c32 is 50, key is test_row_0/C:col10/1732020824207/Put/seqid=0 2024-11-19T12:53:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742306_1482 (size=12561) 2024-11-19T12:53:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:46,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T12:53:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:46,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c7efaeaf1d39485bb2ea0321e639e65b is 50, key is test_row_0/A:col10/1732020825471/Put/seqid=0 2024-11-19T12:53:46,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742307_1483 (size=14541) 2024-11-19T12:53:46,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020886167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020886179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020886281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,291 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8016d40adce742fc9736e4c23a65f37a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8016d40adce742fc9736e4c23a65f37a 2024-11-19T12:53:46,294 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 8016d40adce742fc9736e4c23a65f37a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:46,294 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:46,294 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020825841; duration=0sec 2024-11-19T12:53:46,294 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:46,294 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020886291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,302 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/851b0cd87b9049d6a38b250f82f59c32 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/851b0cd87b9049d6a38b250f82f59c32 2024-11-19T12:53:46,307 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 851b0cd87b9049d6a38b250f82f59c32(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:46,307 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:46,307 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020825841; duration=0sec 2024-11-19T12:53:46,307 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:46,307 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020886491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020886502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c7efaeaf1d39485bb2ea0321e639e65b 2024-11-19T12:53:46,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/de8bf2368569473ba2737df008ba5ab3 is 50, key is test_row_0/B:col10/1732020825471/Put/seqid=0 2024-11-19T12:53:46,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742308_1484 (size=12151) 2024-11-19T12:53:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020886808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020886811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:46,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/de8bf2368569473ba2737df008ba5ab3 2024-11-19T12:53:46,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ef7821bc694e423f93d559992a5e2edc is 50, key is test_row_0/C:col10/1732020825471/Put/seqid=0 2024-11-19T12:53:46,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742309_1485 (size=12151) 2024-11-19T12:53:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T12:53:47,311 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:47,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T12:53:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T12:53:47,313 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:47,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:47,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020887311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:47,319 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:47,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020887321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:47,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020887321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:47,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020887321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:47,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:47,331 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020887322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:47,332 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:47,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ef7821bc694e423f93d559992a5e2edc 2024-11-19T12:53:47,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c7efaeaf1d39485bb2ea0321e639e65b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b 2024-11-19T12:53:47,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b, entries=200, sequenceid=198, filesize=14.2 K 2024-11-19T12:53:47,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/de8bf2368569473ba2737df008ba5ab3 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3 2024-11-19T12:53:47,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T12:53:47,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ef7821bc694e423f93d559992a5e2edc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc 2024-11-19T12:53:47,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T12:53:47,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1266ms, sequenceid=198, compaction requested=false 2024-11-19T12:53:47,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T12:53:47,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T12:53:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:47,469 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:47,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/2f3c7b17248b4e24b062b5f3663f3ac6 is 50, key is test_row_0/A:col10/1732020826153/Put/seqid=0 2024-11-19T12:53:47,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742310_1486 (size=12151) 2024-11-19T12:53:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T12:53:47,879 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/2f3c7b17248b4e24b062b5f3663f3ac6 2024-11-19T12:53:47,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/70c590b851114b308815fce1ba046eaf is 50, key is test_row_0/B:col10/1732020826153/Put/seqid=0 2024-11-19T12:53:47,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742311_1487 (size=12151) 2024-11-19T12:53:47,890 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/70c590b851114b308815fce1ba046eaf 2024-11-19T12:53:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5303fa7bd8c5403fb823d650b1f3cdf0 is 50, key is test_row_0/C:col10/1732020826153/Put/seqid=0 2024-11-19T12:53:47,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742312_1488 (size=12151) 2024-11-19T12:53:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T12:53:48,320 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5303fa7bd8c5403fb823d650b1f3cdf0 2024-11-19T12:53:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/2f3c7b17248b4e24b062b5f3663f3ac6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6 2024-11-19T12:53:48,327 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T12:53:48,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/70c590b851114b308815fce1ba046eaf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf 2024-11-19T12:53:48,330 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T12:53:48,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5303fa7bd8c5403fb823d650b1f3cdf0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0 2024-11-19T12:53:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:48,345 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T12:53:48,346 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 877ms, sequenceid=214, compaction requested=true 2024-11-19T12:53:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T12:53:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T12:53:48,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T12:53:48,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-19T12:53:48,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0370 sec 2024-11-19T12:53:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:48,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:53:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:48,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/7bf9c243033d41a4b48fff6e3dd91885 is 50, key is test_row_0/A:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:48,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742313_1489 (size=14541) 2024-11-19T12:53:48,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/7bf9c243033d41a4b48fff6e3dd91885 2024-11-19T12:53:48,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/a7c2738b7414431e8a4b9c6157e3e159 is 50, key is test_row_0/B:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742314_1490 (size=12151) 2024-11-19T12:53:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T12:53:48,451 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:48,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:48,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020888438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020828395, max timestamp=1732020828452Num uniques 859; 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020828395, max timestamp=1732020828452Num uniques 859; 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020828395, max timestamp=1732020828452Num uniques 859; 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:53:48,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:53:48,455 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:48,456 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:48,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020888587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020888599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T12:53:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020888711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020888711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T12:53:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:48,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/a7c2738b7414431e8a4b9c6157e3e159 2024-11-19T12:53:48,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/c5a2f163747a407482ea03348787b20c is 50, key is test_row_0/C:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:48,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742315_1491 (size=12151) 2024-11-19T12:53:48,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T12:53:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:48,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020888931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020888931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T12:53:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:49,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/c5a2f163747a407482ea03348787b20c 2024-11-19T12:53:49,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/7bf9c243033d41a4b48fff6e3dd91885 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885 2024-11-19T12:53:49,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885, entries=200, sequenceid=225, filesize=14.2 K 2024-11-19T12:53:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/a7c2738b7414431e8a4b9c6157e3e159 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159 2024-11-19T12:53:49,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T12:53:49,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/c5a2f163747a407482ea03348787b20c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c 2024-11-19T12:53:49,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T12:53:49,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 840ms, sequenceid=225, compaction requested=true 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:49,204 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:49,204 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:49,205 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,205 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,205 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8016d40adce742fc9736e4c23a65f37a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=52.5 K 2024-11-19T12:53:49,205 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/65d32696c7b340e398dee65bdba15a75, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=47.9 K 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8016d40adce742fc9736e4c23a65f37a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d32696c7b340e398dee65bdba15a75, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7efaeaf1d39485bb2ea0321e639e65b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732020825451 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting de8bf2368569473ba2737df008ba5ab3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732020825470 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3c7b17248b4e24b062b5f3663f3ac6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732020826153 2024-11-19T12:53:49,205 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c590b851114b308815fce1ba046eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732020826153 2024-11-19T12:53:49,206 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf9c243033d41a4b48fff6e3dd91885, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828354 2024-11-19T12:53:49,206 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c2738b7414431e8a4b9c6157e3e159, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828356 2024-11-19T12:53:49,212 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:49,212 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:49,213 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/2cf2be03698e4dc1a52a1bd3a891e2f0 is 50, key is test_row_0/B:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:49,213 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/7aaa8be3b1c848a3b9e15baf22bd3fcb is 50, key is test_row_0/A:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:49,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T12:53:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,220 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:49,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742317_1493 (size=12697) 2024-11-19T12:53:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742316_1492 (size=12697) 2024-11-19T12:53:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/41b52ef23a094966bfdda3e4f38992fd is 50, key is test_row_0/A:col10/1732020828597/Put/seqid=0 2024-11-19T12:53:49,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742318_1494 (size=12251) 2024-11-19T12:53:49,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:49,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020889301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020889302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020889432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020889432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:49,624 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/2cf2be03698e4dc1a52a1bd3a891e2f0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2cf2be03698e4dc1a52a1bd3a891e2f0 2024-11-19T12:53:49,624 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/7aaa8be3b1c848a3b9e15baf22bd3fcb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7aaa8be3b1c848a3b9e15baf22bd3fcb 2024-11-19T12:53:49,628 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 2cf2be03698e4dc1a52a1bd3a891e2f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:49,628 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:49,628 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=12, startTime=1732020829204; duration=0sec 2024-11-19T12:53:49,628 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:49,628 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:49,628 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:49,629 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:49,629 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:49,629 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:49,629 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/851b0cd87b9049d6a38b250f82f59c32, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=47.9 K 2024-11-19T12:53:49,629 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 7aaa8be3b1c848a3b9e15baf22bd3fcb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:49,629 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:49,630 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=12, startTime=1732020829204; duration=0sec 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 851b0cd87b9049d6a38b250f82f59c32, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732020824197 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7821bc694e423f93d559992a5e2edc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732020825470 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5303fa7bd8c5403fb823d650b1f3cdf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732020826153 2024-11-19T12:53:49,630 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a2f163747a407482ea03348787b20c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828356 2024-11-19T12:53:49,631 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/41b52ef23a094966bfdda3e4f38992fd 2024-11-19T12:53:49,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8e39f7e75ec94964bd70f24167539efc is 50, key is test_row_0/B:col10/1732020828597/Put/seqid=0 2024-11-19T12:53:49,637 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:49,637 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/c072de07271a41b3af8be093faf71d13 is 50, key is test_row_0/C:col10/1732020828362/Put/seqid=0 2024-11-19T12:53:49,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742320_1496 (size=12697) 2024-11-19T12:53:49,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742319_1495 (size=12251) 2024-11-19T12:53:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020889651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020889651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020889961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020889971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,044 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8e39f7e75ec94964bd70f24167539efc 2024-11-19T12:53:50,045 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/c072de07271a41b3af8be093faf71d13 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c072de07271a41b3af8be093faf71d13 2024-11-19T12:53:50,049 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into c072de07271a41b3af8be093faf71d13(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:50,049 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:50,049 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=12, startTime=1732020829204; duration=0sec 2024-11-19T12:53:50,049 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:50,049 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:50,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/b26cf6fcc3df455c9b8cbbcc4ce7079a is 50, key is test_row_0/C:col10/1732020828597/Put/seqid=0 2024-11-19T12:53:50,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742321_1497 (size=12251) 2024-11-19T12:53:50,454 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/b26cf6fcc3df455c9b8cbbcc4ce7079a 2024-11-19T12:53:50,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/41b52ef23a094966bfdda3e4f38992fd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd 2024-11-19T12:53:50,459 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T12:53:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8e39f7e75ec94964bd70f24167539efc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc 2024-11-19T12:53:50,463 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T12:53:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/b26cf6fcc3df455c9b8cbbcc4ce7079a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a 2024-11-19T12:53:50,466 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a, entries=150, sequenceid=261, filesize=12.0 K 2024-11-19T12:53:50,467 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1247ms, sequenceid=261, compaction requested=false 2024-11-19T12:53:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T12:53:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T12:53:50,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T12:53:50,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-11-19T12:53:50,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0190 sec 2024-11-19T12:53:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:50,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:50,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:50,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8e021277af024c0798d0e0aebaaccbdf is 50, key is test_row_0/A:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742322_1498 (size=14741) 2024-11-19T12:53:50,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020890538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020890543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T12:53:50,601 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:50,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T12:53:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:50,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:50,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:50,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:50,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020890652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020890652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:50,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:50,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:50,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020890861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020890862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:50,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:50,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8e021277af024c0798d0e0aebaaccbdf 2024-11-19T12:53:50,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3b62cf6cee9543d199d9692758cf39c0 is 50, key is test_row_0/B:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742323_1499 (size=12301) 2024-11-19T12:53:50,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:50,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:51,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020891171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020891171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:51,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:51,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3b62cf6cee9543d199d9692758cf39c0 2024-11-19T12:53:51,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/de5518342f2a434ea07a16f8c3fc9685 is 50, key is test_row_0/C:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:51,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742324_1500 (size=12301) 2024-11-19T12:53:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58042 deadline: 1732020891361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58074 deadline: 1732020891361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58088 deadline: 1732020891361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:51,364 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_1' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,365 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 1fb4fabd22f8f9ba7eb58c23e43c4dba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:51,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020891691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020891691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:51,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/de5518342f2a434ea07a16f8c3fc9685 2024-11-19T12:53:51,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8e021277af024c0798d0e0aebaaccbdf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf 2024-11-19T12:53:51,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf, entries=200, sequenceid=276, filesize=14.4 K 2024-11-19T12:53:51,719 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T12:53:51,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/3b62cf6cee9543d199d9692758cf39c0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0 2024-11-19T12:53:51,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T12:53:51,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/de5518342f2a434ea07a16f8c3fc9685 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685 2024-11-19T12:53:51,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T12:53:51,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1242ms, sequenceid=276, compaction requested=true 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:51,728 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:51,728 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:51,729 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:51,729 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:51,729 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:51,729 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:51,729 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,729 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,729 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7aaa8be3b1c848a3b9e15baf22bd3fcb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=38.8 K 2024-11-19T12:53:51,729 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2cf2be03698e4dc1a52a1bd3a891e2f0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.4 K 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aaa8be3b1c848a3b9e15baf22bd3fcb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828356 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf2be03698e4dc1a52a1bd3a891e2f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828356 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b52ef23a094966bfdda3e4f38992fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732020828513 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e39f7e75ec94964bd70f24167539efc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732020828513 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e021277af024c0798d0e0aebaaccbdf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829284 2024-11-19T12:53:51,730 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b62cf6cee9543d199d9692758cf39c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829297 2024-11-19T12:53:51,738 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:51,738 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6 is 50, key is test_row_0/B:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:51,740 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#429 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:51,740 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/4ba3d8066d484ba5b127e9f06feb1a3e is 50, key is test_row_0/A:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742325_1501 (size=12949) 2024-11-19T12:53:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742326_1502 (size=12949) 2024-11-19T12:53:51,767 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6 2024-11-19T12:53:51,767 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/4ba3d8066d484ba5b127e9f06feb1a3e as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/4ba3d8066d484ba5b127e9f06feb1a3e 2024-11-19T12:53:51,772 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into e7aff79dcbdc4dcf9f207f9df7c6f5f6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:51,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:51,772 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020831728; duration=0sec 2024-11-19T12:53:51,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:51,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:51,772 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:51,773 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:51,773 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:51,773 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,774 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c072de07271a41b3af8be093faf71d13, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.4 K 2024-11-19T12:53:51,774 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting c072de07271a41b3af8be093faf71d13, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732020828356 2024-11-19T12:53:51,776 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting b26cf6fcc3df455c9b8cbbcc4ce7079a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732020828513 2024-11-19T12:53:51,776 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 4ba3d8066d484ba5b127e9f06feb1a3e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:51,776 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:51,776 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020831728; duration=0sec 2024-11-19T12:53:51,776 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting de5518342f2a434ea07a16f8c3fc9685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829297 2024-11-19T12:53:51,776 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:51,776 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:51,783 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:51,784 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d9dfafa34c1c457099f589ebd50215aa is 50, key is test_row_0/C:col10/1732020829301/Put/seqid=0 2024-11-19T12:53:51,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742327_1503 (size=12949) 2024-11-19T12:53:51,796 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d9dfafa34c1c457099f589ebd50215aa as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d9dfafa34c1c457099f589ebd50215aa 2024-11-19T12:53:51,800 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into d9dfafa34c1c457099f589ebd50215aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:51,800 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:51,800 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020831728; duration=0sec 2024-11-19T12:53:51,800 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:51,800 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:51,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T12:53:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:51,823 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:51,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/3c40a7d957c240e2a4430ea8b878d952 is 50, key is test_row_0/A:col10/1732020830534/Put/seqid=0 2024-11-19T12:53:51,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742328_1504 (size=12301) 2024-11-19T12:53:52,231 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/3c40a7d957c240e2a4430ea8b878d952 2024-11-19T12:53:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/5b59e75784e24b73913121b1f7da9a18 is 50, key is test_row_0/B:col10/1732020830534/Put/seqid=0 2024-11-19T12:53:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742329_1505 (size=12301) 2024-11-19T12:53:52,643 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/5b59e75784e24b73913121b1f7da9a18 2024-11-19T12:53:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/897a1098b3f34903a3fa247d1144830d is 50, key is test_row_0/C:col10/1732020830534/Put/seqid=0 2024-11-19T12:53:52,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742330_1506 (size=12301) 2024-11-19T12:53:52,652 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/897a1098b3f34903a3fa247d1144830d 2024-11-19T12:53:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/3c40a7d957c240e2a4430ea8b878d952 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952 2024-11-19T12:53:52,658 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T12:53:52,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/5b59e75784e24b73913121b1f7da9a18 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18 2024-11-19T12:53:52,662 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T12:53:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/897a1098b3f34903a3fa247d1144830d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d 2024-11-19T12:53:52,665 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T12:53:52,666 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 843ms, sequenceid=299, compaction requested=false 2024-11-19T12:53:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T12:53:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T12:53:52,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T12:53:52,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0640 sec 2024-11-19T12:53:52,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0670 sec 2024-11-19T12:53:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:52,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:52,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8665c3c1ed5e4027a9c00c4d2ee7b119 is 50, key is test_row_0/A:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:52,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742331_1507 (size=14741) 2024-11-19T12:53:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T12:53:52,751 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:52,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T12:53:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:52,753 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:52,753 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:52,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020832755, max timestamp=1732020832772Num uniques 859; 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020832755, max timestamp=1732020832772Num uniques 859; 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020832755, max timestamp=1732020832772Num uniques 859; 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:52,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020892832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:52,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020892834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:52,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:52,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:52,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:52,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020892951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020892951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:53,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:53,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8665c3c1ed5e4027a9c00c4d2ee7b119 2024-11-19T12:53:53,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/9d5a2d1ca6624b33a77c8f962ef21404 is 50, key is test_row_0/B:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:53,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742332_1508 (size=12301) 2024-11-19T12:53:53,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020893161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:53,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020893162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:53,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020893472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020893472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:53,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/9d5a2d1ca6624b33a77c8f962ef21404 2024-11-19T12:53:53,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5a85def3e6b541038656cbd31acea0b7 is 50, key is test_row_0/C:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:53,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742333_1509 (size=12301) 2024-11-19T12:53:53,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:53,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5a85def3e6b541038656cbd31acea0b7 2024-11-19T12:53:53,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/8665c3c1ed5e4027a9c00c4d2ee7b119 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119 2024-11-19T12:53:53,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119, entries=200, sequenceid=309, filesize=14.4 K 2024-11-19T12:53:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/9d5a2d1ca6624b33a77c8f962ef21404 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404 2024-11-19T12:53:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T12:53:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/5a85def3e6b541038656cbd31acea0b7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7 2024-11-19T12:53:53,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T12:53:53,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1242ms, sequenceid=309, compaction requested=true 2024-11-19T12:53:53,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:53,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:53,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:53,957 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:53,958 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,958 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,958 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.7 K 2024-11-19T12:53:53,958 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/4ba3d8066d484ba5b127e9f06feb1a3e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=39.1 K 2024-11-19T12:53:53,958 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e7aff79dcbdc4dcf9f207f9df7c6f5f6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829297 2024-11-19T12:53:53,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ba3d8066d484ba5b127e9f06feb1a3e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829297 2024-11-19T12:53:53,959 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b59e75784e24b73913121b1f7da9a18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732020830530 2024-11-19T12:53:53,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c40a7d957c240e2a4430ea8b878d952, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732020830530 2024-11-19T12:53:53,959 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5a2d1ca6624b33a77c8f962ef21404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832707 2024-11-19T12:53:53,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8665c3c1ed5e4027a9c00c4d2ee7b119, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832702 2024-11-19T12:53:53,964 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:53,964 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8ede4e03acde4c29bf60240f55ab3a0c is 50, key is test_row_0/B:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:53,967 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:53,968 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/e7bee86cfa494df889b1b7bee0e58a0f is 50, key is test_row_0/A:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:53,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742334_1510 (size=13051) 2024-11-19T12:53:53,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742335_1511 (size=13051) 2024-11-19T12:53:53,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T12:53:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:53,976 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:53,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/294a320b239c4f68999a43321149e5d9 is 50, key is test_row_0/A:col10/1732020832833/Put/seqid=0 2024-11-19T12:53:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742336_1512 (size=12301) 2024-11-19T12:53:53,988 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/294a320b239c4f68999a43321149e5d9 2024-11-19T12:53:53,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/42730462b9d4438ababb10b2787088fe is 50, key is test_row_0/B:col10/1732020832833/Put/seqid=0 2024-11-19T12:53:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:53,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:54,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742337_1513 (size=12301) 2024-11-19T12:53:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020894018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020894142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020894361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,375 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8ede4e03acde4c29bf60240f55ab3a0c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8ede4e03acde4c29bf60240f55ab3a0c 2024-11-19T12:53:54,375 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/e7bee86cfa494df889b1b7bee0e58a0f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/e7bee86cfa494df889b1b7bee0e58a0f 2024-11-19T12:53:54,378 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 8ede4e03acde4c29bf60240f55ab3a0c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:54,378 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:54,378 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020833957; duration=0sec 2024-11-19T12:53:54,378 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:54,378 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:54,379 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into e7bee86cfa494df889b1b7bee0e58a0f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:54,379 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020833957; duration=0sec 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:54,379 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:54,379 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:54,379 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d9dfafa34c1c457099f589ebd50215aa, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.7 K 2024-11-19T12:53:54,380 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d9dfafa34c1c457099f589ebd50215aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732020829297 2024-11-19T12:53:54,380 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 897a1098b3f34903a3fa247d1144830d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732020830530 2024-11-19T12:53:54,380 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a85def3e6b541038656cbd31acea0b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832707 2024-11-19T12:53:54,387 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:54,387 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/1d0fecd38fab4556bb2017bf941984a7 is 50, key is test_row_0/C:col10/1732020832711/Put/seqid=0 2024-11-19T12:53:54,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742338_1514 (size=13051) 2024-11-19T12:53:54,402 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/42730462b9d4438ababb10b2787088fe 2024-11-19T12:53:54,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8c4d21f2f53c4fe4a317f5ea1aa43008 is 50, key is test_row_0/C:col10/1732020832833/Put/seqid=0 2024-11-19T12:53:54,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742339_1515 (size=12301) 2024-11-19T12:53:54,421 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8c4d21f2f53c4fe4a317f5ea1aa43008 2024-11-19T12:53:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/294a320b239c4f68999a43321149e5d9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9 2024-11-19T12:53:54,428 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T12:53:54,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/42730462b9d4438ababb10b2787088fe as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe 2024-11-19T12:53:54,433 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T12:53:54,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/8c4d21f2f53c4fe4a317f5ea1aa43008 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008 2024-11-19T12:53:54,437 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008, entries=150, sequenceid=345, filesize=12.0 K 2024-11-19T12:53:54,438 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 462ms, sequenceid=345, compaction requested=false 2024-11-19T12:53:54,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:54,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:54,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T12:53:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T12:53:54,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T12:53:54,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6860 sec 2024-11-19T12:53:54,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6880 sec 2024-11-19T12:53:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:54,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:54,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/843dbfd62cab40ed985c1556776a8762 is 50, key is test_row_0/A:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742340_1516 (size=14741) 2024-11-19T12:53:54,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/843dbfd62cab40ed985c1556776a8762 2024-11-19T12:53:54,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8754731d75a54d04bc89f4720ab64e24 is 50, key is test_row_0/B:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:54,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742341_1517 (size=12301) 2024-11-19T12:53:54,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8754731d75a54d04bc89f4720ab64e24 2024-11-19T12:53:54,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/989bbee4633040a39b23eecba724c252 is 50, key is test_row_0/C:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742342_1518 (size=12301) 2024-11-19T12:53:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020834728, max timestamp=1732020834755Num uniques 859; 2024-11-19T12:53:54,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020834728, max timestamp=1732020834755Num uniques 859; 2024-11-19T12:53:54,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:54,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732020834728, max timestamp=1732020834755Num uniques 859; 2024-11-19T12:53:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T12:53:54,799 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/1d0fecd38fab4556bb2017bf941984a7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/1d0fecd38fab4556bb2017bf941984a7 2024-11-19T12:53:54,818 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 1d0fecd38fab4556bb2017bf941984a7(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:54,818 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:54,818 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020833958; duration=0sec 2024-11-19T12:53:54,818 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:54,818 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020894872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T12:53:54,901 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:54,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T12:53:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:54,904 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:54,905 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:54,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020894992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020894992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:55,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/989bbee4633040a39b23eecba724c252 2024-11-19T12:53:55,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/843dbfd62cab40ed985c1556776a8762 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762 2024-11-19T12:53:55,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762, entries=200, sequenceid=358, filesize=14.4 K 2024-11-19T12:53:55,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/8754731d75a54d04bc89f4720ab64e24 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24 2024-11-19T12:53:55,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T12:53:55,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/989bbee4633040a39b23eecba724c252 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252 2024-11-19T12:53:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T12:53:55,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 447ms, sequenceid=358, compaction requested=true 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:55,130 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:55,130 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:55,131 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,131 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,131 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/e7bee86cfa494df889b1b7bee0e58a0f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=39.2 K 2024-11-19T12:53:55,131 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8ede4e03acde4c29bf60240f55ab3a0c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.8 K 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7bee86cfa494df889b1b7bee0e58a0f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832707 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ede4e03acde4c29bf60240f55ab3a0c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832707 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 294a320b239c4f68999a43321149e5d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732020832812 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 42730462b9d4438ababb10b2787088fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732020832812 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 843dbfd62cab40ed985c1556776a8762, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020833999 2024-11-19T12:53:55,131 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 8754731d75a54d04bc89f4720ab64e24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020834016 2024-11-19T12:53:55,137 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:55,137 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:55,137 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c77f8174acb540b88b3b83b96dbe4958 is 50, key is test_row_0/A:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:55,137 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/56cfc9176ba345cd8cd91c091612f343 is 50, key is test_row_0/B:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:55,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742344_1520 (size=13153) 2024-11-19T12:53:55,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742343_1519 (size=13153) 2024-11-19T12:53:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:55,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:55,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:55,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/a641741edb6742cd976dd8aa0e59f785 is 50, key is test_row_0/A:col10/1732020834870/Put/seqid=0 2024-11-19T12:53:55,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742345_1521 (size=14741) 2024-11-19T12:53:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020895237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020895238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020895351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020895351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:55,546 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/56cfc9176ba345cd8cd91c091612f343 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/56cfc9176ba345cd8cd91c091612f343 2024-11-19T12:53:55,546 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/c77f8174acb540b88b3b83b96dbe4958 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c77f8174acb540b88b3b83b96dbe4958 2024-11-19T12:53:55,550 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into c77f8174acb540b88b3b83b96dbe4958(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:55,550 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:55,550 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=13, startTime=1732020835130; duration=0sec 2024-11-19T12:53:55,550 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:55,550 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:55,550 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:53:55,551 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 56cfc9176ba345cd8cd91c091612f343(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:55,551 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:55,551 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=13, startTime=1732020835130; duration=0sec 2024-11-19T12:53:55,551 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:55,551 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:55,551 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:53:55,551 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:55,551 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,552 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/1d0fecd38fab4556bb2017bf941984a7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=36.8 K 2024-11-19T12:53:55,552 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0fecd38fab4556bb2017bf941984a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732020832707 2024-11-19T12:53:55,552 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c4d21f2f53c4fe4a317f5ea1aa43008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732020832812 2024-11-19T12:53:55,552 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 989bbee4633040a39b23eecba724c252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020834016 2024-11-19T12:53:55,559 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:55,559 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d649894e7dfd4842b1315048d77fe161 is 50, key is test_row_0/C:col10/1732020834016/Put/seqid=0 2024-11-19T12:53:55,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742346_1522 (size=13153) 2024-11-19T12:53:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020895561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020895571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/a641741edb6742cd976dd8aa0e59f785 2024-11-19T12:53:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/e28921cb174a47c594ec679406c61089 is 50, key is test_row_0/B:col10/1732020834870/Put/seqid=0 2024-11-19T12:53:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742347_1523 (size=12301) 2024-11-19T12:53:55,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020895891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:55,967 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/d649894e7dfd4842b1315048d77fe161 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d649894e7dfd4842b1315048d77fe161 2024-11-19T12:53:55,970 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into d649894e7dfd4842b1315048d77fe161(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:55,970 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:55,970 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=13, startTime=1732020835130; duration=0sec 2024-11-19T12:53:55,971 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:55,971 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:55,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:55,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:56,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/e28921cb174a47c594ec679406c61089 2024-11-19T12:53:56,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/3b982064eba145eb9970303e5e8abedd is 50, key is test_row_0/C:col10/1732020834870/Put/seqid=0 2024-11-19T12:53:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742348_1524 (size=12301) 2024-11-19T12:53:56,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/3b982064eba145eb9970303e5e8abedd 2024-11-19T12:53:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:56,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/a641741edb6742cd976dd8aa0e59f785 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785 2024-11-19T12:53:56,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785, entries=200, sequenceid=394, filesize=14.4 K 2024-11-19T12:53:56,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/e28921cb174a47c594ec679406c61089 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089 2024-11-19T12:53:56,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T12:53:56,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/3b982064eba145eb9970303e5e8abedd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd 2024-11-19T12:53:56,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd, entries=150, sequenceid=394, filesize=12.0 K 2024-11-19T12:53:56,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 850ms, sequenceid=394, compaction requested=false 2024-11-19T12:53:56,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:56,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T12:53:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:56,151 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:53:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/605f05d69d084313ba658d7a924cd471 is 50, key is test_row_0/A:col10/1732020835227/Put/seqid=0 2024-11-19T12:53:56,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742349_1525 (size=12301) 2024-11-19T12:53:56,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:56,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020836448, max timestamp=1732020836476Num uniques 859; 2024-11-19T12:53:56,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020836448, max timestamp=1732020836476Num uniques 859; 2024-11-19T12:53:56,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020836448, max timestamp=1732020836476Num uniques 859; 2024-11-19T12:53:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:53:56,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020896544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020896545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,559 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/605f05d69d084313ba658d7a924cd471 2024-11-19T12:53:56,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/10cf4b570f8349a1aae5021f916632f4 is 50, key is test_row_0/B:col10/1732020835227/Put/seqid=0 2024-11-19T12:53:56,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742350_1526 (size=12301) 2024-11-19T12:53:56,569 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/10cf4b570f8349a1aae5021f916632f4 2024-11-19T12:53:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/69748fa98436440891b56c68304a75d9 is 50, key is test_row_0/C:col10/1732020835227/Put/seqid=0 2024-11-19T12:53:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742351_1527 (size=12301) 2024-11-19T12:53:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020896651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020896652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020896871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020896872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:56,979 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/69748fa98436440891b56c68304a75d9 2024-11-19T12:53:56,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/605f05d69d084313ba658d7a924cd471 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471 2024-11-19T12:53:56,984 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471, entries=150, sequenceid=407, filesize=12.0 K 2024-11-19T12:53:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/10cf4b570f8349a1aae5021f916632f4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4 2024-11-19T12:53:56,988 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4, entries=150, sequenceid=407, filesize=12.0 K 2024-11-19T12:53:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/69748fa98436440891b56c68304a75d9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9 2024-11-19T12:53:56,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9, entries=150, sequenceid=407, filesize=12.0 K 2024-11-19T12:53:56,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 841ms, sequenceid=407, compaction requested=true 2024-11-19T12:53:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T12:53:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T12:53:56,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T12:53:56,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0880 sec 2024-11-19T12:53:56,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0950 sec 2024-11-19T12:53:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T12:53:57,051 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:53:57,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:53:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T12:53:57,053 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:53:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:57,054 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:53:57,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:53:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:57,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T12:53:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:57,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:57,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:57,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:57,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:53:57,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/40f2f343f2914b01ba38e0315a904711 is 50, key is test_row_0/A:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742352_1528 (size=14741) 2024-11-19T12:53:57,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020897205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020897205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020897321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020897321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:57,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020897541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020897541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/40f2f343f2914b01ba38e0315a904711 2024-11-19T12:53:57,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/2f9316bd1e09464891e3726e9c63ea45 is 50, key is test_row_0/B:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:57,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742353_1529 (size=12301) 2024-11-19T12:53:57,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:57,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020897861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020897861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:57,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:57,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/2f9316bd1e09464891e3726e9c63ea45 2024-11-19T12:53:58,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/0f702e4b93fb4408b0b3f06c2067e06b is 50, key is test_row_0/C:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:58,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742354_1530 (size=12301) 2024-11-19T12:53:58,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:58,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1732020898381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:53:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58100 deadline: 1732020898381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 , the old value is region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., hostname=aba5a916dfea,35349,1732020694570, seqNum=2 because the exception is null or not the one we care about 2024-11-19T12:53:58,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/0f702e4b93fb4408b0b3f06c2067e06b 2024-11-19T12:53:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/40f2f343f2914b01ba38e0315a904711 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711 2024-11-19T12:53:58,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711, entries=200, sequenceid=444, filesize=14.4 K 2024-11-19T12:53:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/2f9316bd1e09464891e3726e9c63ea45 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45 2024-11-19T12:53:58,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:58,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:53:58,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45, entries=150, sequenceid=444, filesize=12.0 K 2024-11-19T12:53:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/0f702e4b93fb4408b0b3f06c2067e06b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b 2024-11-19T12:53:58,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b, entries=150, sequenceid=444, filesize=12.0 K 2024-11-19T12:53:58,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1245ms, sequenceid=444, compaction requested=true 2024-11-19T12:53:58,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:58,434 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fb4fabd22f8f9ba7eb58c23e43c4dba:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:53:58,434 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:58,434 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:58,434 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/A is initiating minor compaction (all files) 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/B is initiating minor compaction (all files) 2024-11-19T12:53:58,435 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/A in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,435 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/B in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,435 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/56cfc9176ba345cd8cd91c091612f343, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=48.9 K 2024-11-19T12:53:58,435 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c77f8174acb540b88b3b83b96dbe4958, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=53.6 K 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 56cfc9176ba345cd8cd91c091612f343, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020834016 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77f8174acb540b88b3b83b96dbe4958, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020834016 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e28921cb174a47c594ec679406c61089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732020834870 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting a641741edb6742cd976dd8aa0e59f785, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732020834836 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cf4b570f8349a1aae5021f916632f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732020835227 2024-11-19T12:53:58,435 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605f05d69d084313ba658d7a924cd471, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732020835227 2024-11-19T12:53:58,436 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f2f343f2914b01ba38e0315a904711, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732020836527 2024-11-19T12:53:58,436 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9316bd1e09464891e3726e9c63ea45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732020836527 2024-11-19T12:53:58,445 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:58,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/b1d6de2cd1d94825812f28b3f7d553bc is 50, key is test_row_0/B:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:58,453 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:58,454 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/498a854e0d2845c09929f3aa9b5808a7 is 50, key is test_row_0/A:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:58,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742356_1532 (size=13289) 2024-11-19T12:53:58,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742355_1531 (size=13289) 2024-11-19T12:53:58,465 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/498a854e0d2845c09929f3aa9b5808a7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/498a854e0d2845c09929f3aa9b5808a7 2024-11-19T12:53:58,472 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/A of 1fb4fabd22f8f9ba7eb58c23e43c4dba into 498a854e0d2845c09929f3aa9b5808a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:58,472 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:58,472 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/A, priority=12, startTime=1732020838433; duration=0sec 2024-11-19T12:53:58,472 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:53:58,472 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:A 2024-11-19T12:53:58,472 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:53:58,473 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:53:58,473 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 1fb4fabd22f8f9ba7eb58c23e43c4dba/C is initiating minor compaction (all files) 2024-11-19T12:53:58,473 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fb4fabd22f8f9ba7eb58c23e43c4dba/C in TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,473 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d649894e7dfd4842b1315048d77fe161, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp, totalSize=48.9 K 2024-11-19T12:53:58,474 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d649894e7dfd4842b1315048d77fe161, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732020834016 2024-11-19T12:53:58,475 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b982064eba145eb9970303e5e8abedd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732020834870 2024-11-19T12:53:58,475 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69748fa98436440891b56c68304a75d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732020835227 2024-11-19T12:53:58,475 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f702e4b93fb4408b0b3f06c2067e06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732020836527 2024-11-19T12:53:58,484 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fb4fabd22f8f9ba7eb58c23e43c4dba#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:53:58,485 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/f109963ab95c4901aeef31dca5cd4ac8 is 50, key is test_row_0/C:col10/1732020837187/Put/seqid=0 2024-11-19T12:53:58,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742357_1533 (size=13289) 2024-11-19T12:53:58,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T12:53:58,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:58,580 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:53:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:53:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ccda736d14334933a0bd418f49ce20ea is 50, key is test_row_0/A:col10/1732020837203/Put/seqid=0 2024-11-19T12:53:58,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742358_1534 (size=12301) 2024-11-19T12:53:58,588 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ccda736d14334933a0bd418f49ce20ea 2024-11-19T12:53:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/40381e491d974df6b6e2660c1ad17096 is 50, key is test_row_0/B:col10/1732020837203/Put/seqid=0 2024-11-19T12:53:58,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742359_1535 (size=12301) 2024-11-19T12:53:58,708 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-19T12:53:58,708 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:58,708 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,708 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:58,708 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-19T12:53:58,708 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:58,708 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,708 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:58,713 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-19T12:53:58,713 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:58,713 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,713 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:58,714 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-19T12:53:58,714 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:58,714 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,714 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,714 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-19T12:53:58,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:58,714 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:58,714 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,714 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:58,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:58,869 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/b1d6de2cd1d94825812f28b3f7d553bc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1d6de2cd1d94825812f28b3f7d553bc 2024-11-19T12:53:58,876 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/B of 1fb4fabd22f8f9ba7eb58c23e43c4dba into b1d6de2cd1d94825812f28b3f7d553bc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:58,876 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:58,876 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/B, priority=12, startTime=1732020838434; duration=0sec 2024-11-19T12:53:58,876 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:58,876 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:B 2024-11-19T12:53:58,897 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/f109963ab95c4901aeef31dca5cd4ac8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/f109963ab95c4901aeef31dca5cd4ac8 2024-11-19T12:53:58,904 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fb4fabd22f8f9ba7eb58c23e43c4dba/C of 1fb4fabd22f8f9ba7eb58c23e43c4dba into f109963ab95c4901aeef31dca5cd4ac8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:53:58,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:58,904 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., storeName=1fb4fabd22f8f9ba7eb58c23e43c4dba/C, priority=12, startTime=1732020838434; duration=0sec 2024-11-19T12:53:58,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:53:58,904 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fb4fabd22f8f9ba7eb58c23e43c4dba:C 2024-11-19T12:53:59,002 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/40381e491d974df6b6e2660c1ad17096 2024-11-19T12:53:59,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/fccdf16d40794ef3a4d770ac204a3ce7 is 50, key is test_row_0/C:col10/1732020837203/Put/seqid=0 2024-11-19T12:53:59,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742360_1536 (size=12301) 2024-11-19T12:53:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:53:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:53:59,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. as already flushing 2024-11-19T12:53:59,402 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-19T12:53:59,402 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:59,402 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:59,403 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:59,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:59,413 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-19T12:53:59,413 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:53:59,413 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:59,413 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:53:59,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:53:59,417 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/fccdf16d40794ef3a4d770ac204a3ce7 2024-11-19T12:53:59,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/ccda736d14334933a0bd418f49ce20ea as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ccda736d14334933a0bd418f49ce20ea 2024-11-19T12:53:59,424 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ccda736d14334933a0bd418f49ce20ea, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T12:53:59,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/40381e491d974df6b6e2660c1ad17096 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/40381e491d974df6b6e2660c1ad17096 2024-11-19T12:53:59,428 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/40381e491d974df6b6e2660c1ad17096, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T12:53:59,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/fccdf16d40794ef3a4d770ac204a3ce7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/fccdf16d40794ef3a4d770ac204a3ce7 2024-11-19T12:53:59,432 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/fccdf16d40794ef3a4d770ac204a3ce7, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T12:53:59,433 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=13.42 KB/13740 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 853ms, sequenceid=454, compaction requested=false 2024-11-19T12:53:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: 2024-11-19T12:53:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:53:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T12:53:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T12:53:59,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T12:53:59,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3800 sec 2024-11-19T12:53:59,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3840 sec 2024-11-19T12:54:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T12:54:01,212 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:01,393 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-19T12:54:01,393 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:01,393 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,393 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:01,463 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-19T12:54:01,463 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-19T12:54:01,463 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:01,463 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:01,463 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,463 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,463 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,463 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:01,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:01,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:54:01,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1114 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3342 rows 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1111 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3330 rows 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1108 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3323 rows 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1113 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3333 rows 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1111 2024-11-19T12:54:01,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3329 rows 2024-11-19T12:54:01,464 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:01,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:01,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:01,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:01,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:54:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:01,467 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020841467"}]},"ts":"1732020841467"} 2024-11-19T12:54:01,469 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:54:01,469 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:54:01,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:54:01,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, UNASSIGN}] 2024-11-19T12:54:01,472 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, UNASSIGN 2024-11-19T12:54:01,472 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1fb4fabd22f8f9ba7eb58c23e43c4dba, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:01,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, UNASSIGN because future has completed 2024-11-19T12:54:01,474 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:54:01,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:54:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:01,627 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:54:01,627 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:54:01,627 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 1fb4fabd22f8f9ba7eb58c23e43c4dba, disabling compactions & flushes 2024-11-19T12:54:01,627 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:54:01,627 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:54:01,627 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. after waiting 0 ms 2024-11-19T12:54:01,627 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:54:01,628 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=A 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=B 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fb4fabd22f8f9ba7eb58c23e43c4dba, store=C 2024-11-19T12:54:01,628 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:01,631 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/39a55eefbc95437cadb056155834a2ca is 50, key is test_row_0/A:col10/1732020841462/Put/seqid=0 2024-11-19T12:54:01,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742361_1537 (size=12301) 2024-11-19T12:54:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:02,035 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/39a55eefbc95437cadb056155834a2ca 2024-11-19T12:54:02,041 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/45fc180348de461885bc042181353a82 is 50, key is test_row_0/B:col10/1732020841462/Put/seqid=0 2024-11-19T12:54:02,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742362_1538 (size=12301) 2024-11-19T12:54:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:02,448 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/45fc180348de461885bc042181353a82 2024-11-19T12:54:02,455 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ebe7cf6b88d14799b104c47fa1c270b5 is 50, key is test_row_0/C:col10/1732020841462/Put/seqid=0 2024-11-19T12:54:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742363_1539 (size=12301) 2024-11-19T12:54:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:02,859 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ebe7cf6b88d14799b104c47fa1c270b5 2024-11-19T12:54:02,863 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/A/39a55eefbc95437cadb056155834a2ca as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/39a55eefbc95437cadb056155834a2ca 2024-11-19T12:54:02,865 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/39a55eefbc95437cadb056155834a2ca, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T12:54:02,866 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/B/45fc180348de461885bc042181353a82 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/45fc180348de461885bc042181353a82 2024-11-19T12:54:02,869 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/45fc180348de461885bc042181353a82, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T12:54:02,870 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/.tmp/C/ebe7cf6b88d14799b104c47fa1c270b5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ebe7cf6b88d14799b104c47fa1c270b5 2024-11-19T12:54:02,873 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ebe7cf6b88d14799b104c47fa1c270b5, entries=150, sequenceid=464, filesize=12.0 K 2024-11-19T12:54:02,873 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1246ms, sequenceid=464, compaction requested=true 2024-11-19T12:54:02,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8bb5ec7ff87848f393fbcab1c2a3dd52, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/fb6b080c3bf8407785958e6c76cf2239, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2c78a7cc6ed7497d8f583a6df633bfc0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8016d40adce742fc9736e4c23a65f37a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7aaa8be3b1c848a3b9e15baf22bd3fcb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/4ba3d8066d484ba5b127e9f06feb1a3e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/e7bee86cfa494df889b1b7bee0e58a0f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c77f8174acb540b88b3b83b96dbe4958, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711] to archive 2024-11-19T12:54:02,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:02,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ef0fa5d47a5040fdb6e825e3e398780e 2024-11-19T12:54:02,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c50519523e6d474284184cdf67530e30 2024-11-19T12:54:02,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/1341662044ec429fb4d8a7a5bd8878aa 2024-11-19T12:54:02,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8bb5ec7ff87848f393fbcab1c2a3dd52 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8bb5ec7ff87848f393fbcab1c2a3dd52 2024-11-19T12:54:02,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ce4ef40c574f429886f0f878e4bd1b95 2024-11-19T12:54:02,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/fb6b080c3bf8407785958e6c76cf2239 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/fb6b080c3bf8407785958e6c76cf2239 2024-11-19T12:54:02,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/63a3006354a846959746c10b3ce24d88 2024-11-19T12:54:02,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/850be3efd72c40ea920c54619332637e 2024-11-19T12:54:02,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8be600bbc63b43efb6a8917e7df11ebb 2024-11-19T12:54:02,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2c78a7cc6ed7497d8f583a6df633bfc0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2c78a7cc6ed7497d8f583a6df633bfc0 2024-11-19T12:54:02,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/449a0bbfdafb456c83544a8116d90c10 2024-11-19T12:54:02,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/74f57e23777444fcaf85ef47e852ea69 2024-11-19T12:54:02,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8016d40adce742fc9736e4c23a65f37a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8016d40adce742fc9736e4c23a65f37a 2024-11-19T12:54:02,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c7efaeaf1d39485bb2ea0321e639e65b 2024-11-19T12:54:02,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/2f3c7b17248b4e24b062b5f3663f3ac6 2024-11-19T12:54:02,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7bf9c243033d41a4b48fff6e3dd91885 2024-11-19T12:54:02,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7aaa8be3b1c848a3b9e15baf22bd3fcb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/7aaa8be3b1c848a3b9e15baf22bd3fcb 2024-11-19T12:54:02,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/41b52ef23a094966bfdda3e4f38992fd 2024-11-19T12:54:02,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8e021277af024c0798d0e0aebaaccbdf 2024-11-19T12:54:02,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/4ba3d8066d484ba5b127e9f06feb1a3e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/4ba3d8066d484ba5b127e9f06feb1a3e 2024-11-19T12:54:02,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/3c40a7d957c240e2a4430ea8b878d952 2024-11-19T12:54:02,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/8665c3c1ed5e4027a9c00c4d2ee7b119 2024-11-19T12:54:02,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/e7bee86cfa494df889b1b7bee0e58a0f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/e7bee86cfa494df889b1b7bee0e58a0f 2024-11-19T12:54:02,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/294a320b239c4f68999a43321149e5d9 2024-11-19T12:54:02,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/843dbfd62cab40ed985c1556776a8762 2024-11-19T12:54:02,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c77f8174acb540b88b3b83b96dbe4958 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/c77f8174acb540b88b3b83b96dbe4958 2024-11-19T12:54:02,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/a641741edb6742cd976dd8aa0e59f785 2024-11-19T12:54:02,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/605f05d69d084313ba658d7a924cd471 2024-11-19T12:54:02,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/40f2f343f2914b01ba38e0315a904711 2024-11-19T12:54:02,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/05c27b07687e4ee4bf75505b98907b66, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/147bdb43e9bf4043aaf94f4d7f9c6afb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/84bb9befbdd6474f8e4a399ec443783e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/65d32696c7b340e398dee65bdba15a75, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2cf2be03698e4dc1a52a1bd3a891e2f0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8ede4e03acde4c29bf60240f55ab3a0c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/56cfc9176ba345cd8cd91c091612f343, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45] to archive 2024-11-19T12:54:02,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:02,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/66a22284c7524981ae27ebcd2608930c 2024-11-19T12:54:02,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ebc493da1de4c11b7d153b29d74ec30 2024-11-19T12:54:02,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/05c27b07687e4ee4bf75505b98907b66 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/05c27b07687e4ee4bf75505b98907b66 2024-11-19T12:54:02,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/cdce841b5d064016afd7f77530b80fb7 2024-11-19T12:54:02,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/04f7371db4ac472dae949d66c7041175 2024-11-19T12:54:02,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/147bdb43e9bf4043aaf94f4d7f9c6afb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/147bdb43e9bf4043aaf94f4d7f9c6afb 2024-11-19T12:54:02,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/c8c2a9af5d28480f9f08e9a4a4126d72 2024-11-19T12:54:02,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/4ec7dfe6b1934ad4ad2d8f3511aa5c3d 2024-11-19T12:54:02,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/84bb9befbdd6474f8e4a399ec443783e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/84bb9befbdd6474f8e4a399ec443783e 2024-11-19T12:54:02,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1c8df9633a84daa92ad269eed3783a9 2024-11-19T12:54:02,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/f5a1e235eceb4ee1b7dbcc8cde16cf1b 2024-11-19T12:54:02,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/65d32696c7b340e398dee65bdba15a75 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/65d32696c7b340e398dee65bdba15a75 2024-11-19T12:54:02,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3ea97a19c40a48cb9410a863f950bdfc 2024-11-19T12:54:02,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/de8bf2368569473ba2737df008ba5ab3 2024-11-19T12:54:02,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/70c590b851114b308815fce1ba046eaf 2024-11-19T12:54:02,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2cf2be03698e4dc1a52a1bd3a891e2f0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2cf2be03698e4dc1a52a1bd3a891e2f0 2024-11-19T12:54:02,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/a7c2738b7414431e8a4b9c6157e3e159 2024-11-19T12:54:02,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8e39f7e75ec94964bd70f24167539efc 2024-11-19T12:54:02,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e7aff79dcbdc4dcf9f207f9df7c6f5f6 2024-11-19T12:54:02,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/3b62cf6cee9543d199d9692758cf39c0 2024-11-19T12:54:02,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/5b59e75784e24b73913121b1f7da9a18 2024-11-19T12:54:02,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8ede4e03acde4c29bf60240f55ab3a0c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8ede4e03acde4c29bf60240f55ab3a0c 2024-11-19T12:54:02,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/9d5a2d1ca6624b33a77c8f962ef21404 2024-11-19T12:54:02,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/42730462b9d4438ababb10b2787088fe 2024-11-19T12:54:02,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/56cfc9176ba345cd8cd91c091612f343 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/56cfc9176ba345cd8cd91c091612f343 2024-11-19T12:54:02,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/8754731d75a54d04bc89f4720ab64e24 2024-11-19T12:54:02,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/e28921cb174a47c594ec679406c61089 2024-11-19T12:54:02,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/10cf4b570f8349a1aae5021f916632f4 2024-11-19T12:54:02,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/2f9316bd1e09464891e3726e9c63ea45 2024-11-19T12:54:02,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/cc9d1b8d16004ab3abcdf10a15d25ced, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/aad06023433044e2a89676a87e4f6ccd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/6233694980ed49afa772df09397ac40e, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/851b0cd87b9049d6a38b250f82f59c32, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c072de07271a41b3af8be093faf71d13, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d9dfafa34c1c457099f589ebd50215aa, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/1d0fecd38fab4556bb2017bf941984a7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d649894e7dfd4842b1315048d77fe161, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b] to archive 2024-11-19T12:54:02,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:02,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/40d08e2957f04aeca09c5c8a403548c8 2024-11-19T12:54:02,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ab6e7f7a06244125a8a4fcf2b12317b8 2024-11-19T12:54:02,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/cc9d1b8d16004ab3abcdf10a15d25ced to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/cc9d1b8d16004ab3abcdf10a15d25ced 2024-11-19T12:54:02,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/2f67b891de6f4089989048b7a21fa77c 2024-11-19T12:54:02,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/afb059673f80437e813e4bc00acddcc0 2024-11-19T12:54:02,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/aad06023433044e2a89676a87e4f6ccd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/aad06023433044e2a89676a87e4f6ccd 2024-11-19T12:54:02,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/4f0ef5050e69466dba4f169270403005 2024-11-19T12:54:02,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/75cb393378e14882b38bb05145132f44 2024-11-19T12:54:02,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/6233694980ed49afa772df09397ac40e to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/6233694980ed49afa772df09397ac40e 2024-11-19T12:54:02,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8bd11e3bb1a54852814f1803068805fb 2024-11-19T12:54:02,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d1e70ac1b2e6408aa5858c1028f54288 2024-11-19T12:54:02,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/851b0cd87b9049d6a38b250f82f59c32 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/851b0cd87b9049d6a38b250f82f59c32 2024-11-19T12:54:02,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/44f2c127f0914b868b66b24aed904686 2024-11-19T12:54:02,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ef7821bc694e423f93d559992a5e2edc 2024-11-19T12:54:02,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5303fa7bd8c5403fb823d650b1f3cdf0 2024-11-19T12:54:02,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c072de07271a41b3af8be093faf71d13 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c072de07271a41b3af8be093faf71d13 2024-11-19T12:54:02,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/c5a2f163747a407482ea03348787b20c 2024-11-19T12:54:02,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/b26cf6fcc3df455c9b8cbbcc4ce7079a 2024-11-19T12:54:02,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d9dfafa34c1c457099f589ebd50215aa to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d9dfafa34c1c457099f589ebd50215aa 2024-11-19T12:54:02,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/de5518342f2a434ea07a16f8c3fc9685 2024-11-19T12:54:02,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/897a1098b3f34903a3fa247d1144830d 2024-11-19T12:54:02,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/1d0fecd38fab4556bb2017bf941984a7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/1d0fecd38fab4556bb2017bf941984a7 2024-11-19T12:54:02,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/5a85def3e6b541038656cbd31acea0b7 2024-11-19T12:54:02,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/8c4d21f2f53c4fe4a317f5ea1aa43008 2024-11-19T12:54:02,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d649894e7dfd4842b1315048d77fe161 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/d649894e7dfd4842b1315048d77fe161 2024-11-19T12:54:02,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/989bbee4633040a39b23eecba724c252 2024-11-19T12:54:02,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/3b982064eba145eb9970303e5e8abedd 2024-11-19T12:54:02,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/69748fa98436440891b56c68304a75d9 2024-11-19T12:54:02,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/0f702e4b93fb4408b0b3f06c2067e06b 2024-11-19T12:54:02,952 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/recovered.edits/467.seqid, newMaxSeqId=467, maxSeqId=1 2024-11-19T12:54:02,953 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. 2024-11-19T12:54:02,953 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 1fb4fabd22f8f9ba7eb58c23e43c4dba: Waiting for close lock at 1732020841627Running coprocessor pre-close hooks at 1732020841627Disabling compacts and flushes for region at 1732020841627Disabling writes for close at 1732020841627Obtaining lock to block concurrent updates at 1732020841628 (+1 ms)Preparing flush snapshotting stores in 1fb4fabd22f8f9ba7eb58c23e43c4dba at 1732020841628Finished memstore snapshotting TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732020841628Flushing stores of TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba. at 1732020841628Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/A: creating writer at 1732020841628Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/A: appending metadata at 1732020841631 (+3 ms)Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/A: closing flushed file at 1732020841631Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/B: creating writer at 1732020842038 (+407 ms)Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/B: appending metadata at 1732020842041 (+3 ms)Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/B: closing flushed file at 1732020842041Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/C: creating writer at 1732020842452 (+411 ms)Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/C: appending metadata at 1732020842455 (+3 ms)Flushing 1fb4fabd22f8f9ba7eb58c23e43c4dba/C: closing flushed file at 1732020842455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b76f7b: reopening flushed file at 1732020842862 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5faf626f: reopening flushed file at 1732020842866 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35b25511: reopening flushed file at 1732020842869 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1fb4fabd22f8f9ba7eb58c23e43c4dba in 1246ms, sequenceid=464, compaction requested=true at 1732020842873 (+4 ms)Writing region close event to WAL at 1732020842950 (+77 ms)Running coprocessor post-close hooks at 1732020842953 (+3 ms)Closed at 1732020842953 2024-11-19T12:54:02,955 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:54:02,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1fb4fabd22f8f9ba7eb58c23e43c4dba, regionState=CLOSED 2024-11-19T12:54:02,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:54:02,958 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:54:02,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T12:54:02,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fb4fabd22f8f9ba7eb58c23e43c4dba, server=aba5a916dfea,35349,1732020694570 in 1.4830 sec 2024-11-19T12:54:02,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-19T12:54:02,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fb4fabd22f8f9ba7eb58c23e43c4dba, UNASSIGN in 1.4870 sec 2024-11-19T12:54:02,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T12:54:02,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4900 sec 2024-11-19T12:54:02,963 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020842963"}]},"ts":"1732020842963"} 2024-11-19T12:54:02,964 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:54:02,964 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:54:02,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5000 sec 2024-11-19T12:54:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T12:54:03,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:03,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:54:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,614 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T12:54:03,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:54:03,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/recovered.edits] 2024-11-19T12:54:03,620 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/39a55eefbc95437cadb056155834a2ca to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/39a55eefbc95437cadb056155834a2ca 2024-11-19T12:54:03,621 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/498a854e0d2845c09929f3aa9b5808a7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/498a854e0d2845c09929f3aa9b5808a7 2024-11-19T12:54:03,623 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ccda736d14334933a0bd418f49ce20ea to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/A/ccda736d14334933a0bd418f49ce20ea 2024-11-19T12:54:03,625 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/40381e491d974df6b6e2660c1ad17096 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/40381e491d974df6b6e2660c1ad17096 2024-11-19T12:54:03,626 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/45fc180348de461885bc042181353a82 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/45fc180348de461885bc042181353a82 2024-11-19T12:54:03,627 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1d6de2cd1d94825812f28b3f7d553bc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/B/b1d6de2cd1d94825812f28b3f7d553bc 2024-11-19T12:54:03,628 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ebe7cf6b88d14799b104c47fa1c270b5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/ebe7cf6b88d14799b104c47fa1c270b5 2024-11-19T12:54:03,629 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/f109963ab95c4901aeef31dca5cd4ac8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/f109963ab95c4901aeef31dca5cd4ac8 2024-11-19T12:54:03,630 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/fccdf16d40794ef3a4d770ac204a3ce7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/C/fccdf16d40794ef3a4d770ac204a3ce7 2024-11-19T12:54:03,632 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/recovered.edits/467.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba/recovered.edits/467.seqid 2024-11-19T12:54:03,633 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/1fb4fabd22f8f9ba7eb58c23e43c4dba 2024-11-19T12:54:03,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:54:03,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,637 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:54:03,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:54:03,641 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,641 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:54:03,642 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020843641"}]},"ts":"9223372036854775807"} 2024-11-19T12:54:03,644 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:54:03,644 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1fb4fabd22f8f9ba7eb58c23e43c4dba, NAME => 'TestAcidGuarantees,,1732020816510.1fb4fabd22f8f9ba7eb58c23e43c4dba.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:54:03,644 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:54:03,645 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020843644"}]},"ts":"9223372036854775807"} 2024-11-19T12:54:03,647 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:54:03,648 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-19T12:54:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T12:54:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:54:03,724 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:03,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=514 (was 529), ProcessCount=11 (was 11), AvailableMemoryMB=6796 (was 5425) - AvailableMemoryMB LEAK? - 2024-11-19T12:54:03,742 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=514, ProcessCount=11, AvailableMemoryMB=6796 2024-11-19T12:54:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:54:03,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:03,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:54:03,745 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:03,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-19T12:54:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:03,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:54:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742364_1540 (size=1061) 2024-11-19T12:54:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:04,152 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27 2024-11-19T12:54:04,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742365_1541 (size=53) 2024-11-19T12:54:04,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T12:54:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9e9d5d81a38b9f7fdc417436276ba9a3, disabling compactions & flushes 2024-11-19T12:54:04,561 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. after waiting 0 ms 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,561 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e9d5d81a38b9f7fdc417436276ba9a3: Waiting for close lock at 1732020844561Disabling compacts and flushes for region at 1732020844561Disabling writes for close at 1732020844561Writing region close event to WAL at 1732020844561Closed at 1732020844561 2024-11-19T12:54:04,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:54:04,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732020844562"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020844562"}]},"ts":"1732020844562"} 2024-11-19T12:54:04,564 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:54:04,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:54:04,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020844565"}]},"ts":"1732020844565"} 2024-11-19T12:54:04,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T12:54:04,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, ASSIGN}] 2024-11-19T12:54:04,567 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, ASSIGN 2024-11-19T12:54:04,568 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, ASSIGN; state=OFFLINE, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=false 2024-11-19T12:54:04,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:04,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, ASSIGN because future has completed 2024-11-19T12:54:04,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:54:04,875 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:04,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:04,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,876 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,878 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:04,878 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName A 2024-11-19T12:54:04,878 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:04,879 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:04,879 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,879 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:04,880 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName B 2024-11-19T12:54:04,880 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:04,880 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:04,880 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:04,881 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:04,881 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName C 2024-11-19T12:54:04,881 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:04,882 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:04,882 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,882 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,882 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,882 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,883 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,883 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,884 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:54:04,885 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,887 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:04,887 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 9e9d5d81a38b9f7fdc417436276ba9a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59079230, jitterRate=-0.11965087056159973}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:54:04,887 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:04,887 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 9e9d5d81a38b9f7fdc417436276ba9a3: Running coprocessor pre-open hook at 1732020844875Writing region info on filesystem at 1732020844875Initializing all the Stores at 1732020844876 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020844876Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020844876Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020844876Cleaning up temporary data from old regions at 1732020844883 (+7 ms)Running coprocessor post-open hooks at 1732020844887 (+4 ms)Region opened successfully at 1732020844887 2024-11-19T12:54:04,888 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., pid=144, masterSystemTime=1732020844872 2024-11-19T12:54:04,889 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,889 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:04,890 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:04,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:54:04,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T12:54:04,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 in 171 msec 2024-11-19T12:54:04,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T12:54:04,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, ASSIGN in 327 msec 2024-11-19T12:54:04,894 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:54:04,895 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020844894"}]},"ts":"1732020844894"} 2024-11-19T12:54:04,895 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T12:54:04,896 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:54:04,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-19T12:54:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-19T12:54:05,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:05,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:05,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:05,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:05,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c28d09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:05,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bb6297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:05,896 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:05,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:05,898 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:54:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36409,1732020693843 2024-11-19T12:54:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f99347 2024-11-19T12:54:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:54:05,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:54:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T12:54:05,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742366_1542 (size=1097) 2024-11-19T12:54:06,313 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T12:54:06,313 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T12:54:06,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:54:06,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE}] 2024-11-19T12:54:06,317 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE 2024-11-19T12:54:06,317 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE because future has completed 2024-11-19T12:54:06,319 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:54:06,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:54:06,471 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,471 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:54:06,471 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 9e9d5d81a38b9f7fdc417436276ba9a3, disabling compactions & flushes 2024-11-19T12:54:06,471 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,471 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,471 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. after waiting 0 ms 2024-11-19T12:54:06,471 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,474 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:54:06,474 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,474 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 9e9d5d81a38b9f7fdc417436276ba9a3: Waiting for close lock at 1732020846471Running coprocessor pre-close hooks at 1732020846471Disabling compacts and flushes for region at 1732020846471Disabling writes for close at 1732020846471Writing region close event to WAL at 1732020846471Running coprocessor post-close hooks at 1732020846474 (+3 ms)Closed at 1732020846474 2024-11-19T12:54:06,474 WARN [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 9e9d5d81a38b9f7fdc417436276ba9a3 to self. 2024-11-19T12:54:06,476 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,476 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=CLOSED 2024-11-19T12:54:06,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:54:06,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T12:54:06,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 in 159 msec 2024-11-19T12:54:06,480 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE; state=CLOSED, location=aba5a916dfea,35349,1732020694570; forceNewPlan=false, retain=true 2024-11-19T12:54:06,631 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=OPENING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE because future has completed 2024-11-19T12:54:06,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:54:06,786 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,786 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:06,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:06,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,788 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,788 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:06,789 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName A 2024-11-19T12:54:06,789 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:06,790 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:06,790 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,790 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:06,790 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName B 2024-11-19T12:54:06,790 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:06,791 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:06,791 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,791 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T12:54:06,791 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e9d5d81a38b9f7fdc417436276ba9a3 columnFamilyName C 2024-11-19T12:54:06,791 DEBUG [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:06,791 INFO [StoreOpener-9e9d5d81a38b9f7fdc417436276ba9a3-1 {}] regionserver.HStore(327): Store=9e9d5d81a38b9f7fdc417436276ba9a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:06,792 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,792 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,792 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,793 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,794 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,794 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,794 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T12:54:06,795 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,796 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 9e9d5d81a38b9f7fdc417436276ba9a3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60668123, jitterRate=-0.09597451984882355}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T12:54:06,796 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,797 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 9e9d5d81a38b9f7fdc417436276ba9a3: Running coprocessor pre-open hook at 1732020846787Writing region info on filesystem at 1732020846787Initializing all the Stores at 1732020846788 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020846788Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020846788Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020846788Cleaning up temporary data from old regions at 1732020846794 (+6 ms)Running coprocessor post-open hooks at 1732020846796 (+2 ms)Region opened successfully at 1732020846797 (+1 ms) 2024-11-19T12:54:06,797 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., pid=149, masterSystemTime=1732020846784 2024-11-19T12:54:06,799 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,799 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,800 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=OPEN, openSeqNum=5, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:54:06,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-19T12:54:06,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 in 169 msec 2024-11-19T12:54:06,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T12:54:06,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, REOPEN/MOVE in 488 msec 2024-11-19T12:54:06,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-19T12:54:06,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-19T12:54:06,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-19T12:54:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T12:54:06,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:06,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa39191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c714d37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d1cdce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,815 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd3c12a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,817 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@165d18a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14740061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a664b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,821 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3b79ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a95bc9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,823 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211d49e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687fdbef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b324d95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,828 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41441495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b851b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4d6907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,832 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7721ac58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f81ee90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623f2b91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36409,-1 for getting cluster id 2024-11-19T12:54:06,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:06,837 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f46e4652-6752-4a3d-82ab-95e142937d03' 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f46e4652-6752-4a3d-82ab-95e142937d03" 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac3ab9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36409,-1] 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:06,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e965975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:06,839 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,839 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,839 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,839 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:06,840 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,840 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T12:54:06,841 DEBUG [Thread-2431 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,841 DEBUG [Thread-2431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:06,842 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:06,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,846 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,847 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,847 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,35349,1732020694570, seqNum=-1] 2024-11-19T12:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:06,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:54:06,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5] 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:06,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020846877, max timestamp=1732020846882Num uniques 859; 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020846877, max timestamp=1732020846882Num uniques 859; 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020846877, max timestamp=1732020846882Num uniques 859; 2024-11-19T12:54:06,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:54:06,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119da5325c4123f4122a89b87532f38746f_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020846850/Put/seqid=0 2024-11-19T12:54:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742367_1543 (size=9714) 2024-11-19T12:54:06,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:06,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119da5325c4123f4122a89b87532f38746f_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119da5325c4123f4122a89b87532f38746f_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:06,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/79620c87ad3d4993b66d353605242b64, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:06,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/79620c87ad3d4993b66d353605242b64 is 175, key is test_row_0/A:col10/1732020846850/Put/seqid=0 2024-11-19T12:54:06,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742368_1544 (size=22361) 2024-11-19T12:54:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020906905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020906907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:06,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:06,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:06,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:06,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020907051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020907051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020907061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020907061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020907062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020907261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020907262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020907271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020907271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020907271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/79620c87ad3d4993b66d353605242b64 2024-11-19T12:54:07,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/5aa5bc48722241428b9519e55271b105 is 50, key is test_row_0/B:col10/1732020846850/Put/seqid=0 2024-11-19T12:54:07,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742369_1545 (size=9657) 2024-11-19T12:54:07,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/5aa5bc48722241428b9519e55271b105 2024-11-19T12:54:07,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/7d3c58624c8f4fab9ed2f54033a96023 is 50, key is test_row_0/C:col10/1732020846850/Put/seqid=0 2024-11-19T12:54:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742370_1546 (size=9657) 2024-11-19T12:54:07,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/7d3c58624c8f4fab9ed2f54033a96023 2024-11-19T12:54:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/79620c87ad3d4993b66d353605242b64 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64 2024-11-19T12:54:07,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64, entries=100, sequenceid=15, filesize=21.8 K 2024-11-19T12:54:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/5aa5bc48722241428b9519e55271b105 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105 2024-11-19T12:54:07,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105, entries=100, sequenceid=15, filesize=9.4 K 2024-11-19T12:54:07,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/7d3c58624c8f4fab9ed2f54033a96023 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023 2024-11-19T12:54:07,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023, entries=100, sequenceid=15, filesize=9.4 K 2024-11-19T12:54:07,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=138.65 KB/141980 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 617ms, sequenceid=15, compaction requested=false 2024-11-19T12:54:07,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:07,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=152.07 KB heapSize=435.69 KB 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:07,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b35479b9a86449d7a9dd02d6e6ff7179_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020847574/Put/seqid=0 2024-11-19T12:54:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020907583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020907583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020907588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020907590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020907596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742371_1547 (size=17034) 2024-11-19T12:54:07,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:07,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b35479b9a86449d7a9dd02d6e6ff7179_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b35479b9a86449d7a9dd02d6e6ff7179_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:07,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b7079c4c24324ecc97fd372f7320b173, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:07,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b7079c4c24324ecc97fd372f7320b173 is 175, key is test_row_0/A:col10/1732020847574/Put/seqid=0 2024-11-19T12:54:07,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742372_1548 (size=48139) 2024-11-19T12:54:07,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b7079c4c24324ecc97fd372f7320b173 2024-11-19T12:54:07,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/bf628a34eced42a997ff83d05f8f0284 is 50, key is test_row_0/B:col10/1732020847574/Put/seqid=0 2024-11-19T12:54:07,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742373_1549 (size=12001) 2024-11-19T12:54:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020907701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020907702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020907702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020907713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020907713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020907911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020907912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020907912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:07,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020907921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020907921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:08,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/bf628a34eced42a997ff83d05f8f0284 2024-11-19T12:54:08,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1bbf1af348774eaf803d4d9089919035 is 50, key is test_row_0/C:col10/1732020847574/Put/seqid=0 2024-11-19T12:54:08,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742374_1550 (size=12001) 2024-11-19T12:54:08,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020908222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020908222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020908222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020908231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020908231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:08,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:08,475 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:54:08,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1bbf1af348774eaf803d4d9089919035 2024-11-19T12:54:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b7079c4c24324ecc97fd372f7320b173 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173 2024-11-19T12:54:08,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173, entries=250, sequenceid=54, filesize=47.0 K 2024-11-19T12:54:08,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/bf628a34eced42a997ff83d05f8f0284 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284 2024-11-19T12:54:08,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T12:54:08,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1bbf1af348774eaf803d4d9089919035 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035 2024-11-19T12:54:08,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T12:54:08,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~435.64 KB/446096, currentSize=33.54 KB/34350 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 938ms, sequenceid=54, compaction requested=false 2024-11-19T12:54:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:08,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194cd270f201de47ce8dca843abbf65137_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020847581/Put/seqid=0 2024-11-19T12:54:08,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742375_1551 (size=12154) 2024-11-19T12:54:08,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:08,540 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194cd270f201de47ce8dca843abbf65137_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cd270f201de47ce8dca843abbf65137_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b0a7eddf24ed431984db5344980c3949, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b0a7eddf24ed431984db5344980c3949 is 175, key is test_row_0/A:col10/1732020847581/Put/seqid=0 2024-11-19T12:54:08,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742376_1552 (size=30955) 2024-11-19T12:54:08,546 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b0a7eddf24ed431984db5344980c3949 2024-11-19T12:54:08,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a1883d0c12d544a8a6cbef5175cf4e54 is 50, key is test_row_0/B:col10/1732020847581/Put/seqid=0 2024-11-19T12:54:08,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742377_1553 (size=12001) 2024-11-19T12:54:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:08,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020848747, max timestamp=1732020848757Num uniques 859; 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020848747, max timestamp=1732020848757Num uniques 859; 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732020848747, max timestamp=1732020848757Num uniques 859; 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:54:08,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T12:54:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020908782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020908783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020908783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020908785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020908786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020908901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020908901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020908902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020908902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020908902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:08,970 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a1883d0c12d544a8a6cbef5175cf4e54 2024-11-19T12:54:08,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/26f978960047469d93d959b03fb30527 is 50, key is test_row_0/C:col10/1732020847581/Put/seqid=0 2024-11-19T12:54:08,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742378_1554 (size=12001) 2024-11-19T12:54:08,983 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/26f978960047469d93d959b03fb30527 2024-11-19T12:54:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b0a7eddf24ed431984db5344980c3949 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949 2024-11-19T12:54:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949, entries=150, sequenceid=62, filesize=30.2 K 2024-11-19T12:54:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a1883d0c12d544a8a6cbef5175cf4e54 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54 2024-11-19T12:54:08,996 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T12:54:08,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/26f978960047469d93d959b03fb30527 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527 2024-11-19T12:54:09,002 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T12:54:09,003 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 476ms, sequenceid=62, compaction requested=true 2024-11-19T12:54:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T12:54:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T12:54:09,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T12:54:09,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-19T12:54:09,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.1670 sec 2024-11-19T12:54:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:09,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:09,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:09,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f76829d1d94f40e69076efe08615da9b_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020909117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020909121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020909121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020909125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020909125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742379_1555 (size=12154) 2024-11-19T12:54:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020909242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020909242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020909242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020909242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020909452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020909452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020909452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020909452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020909452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,539 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:09,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f76829d1d94f40e69076efe08615da9b_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f76829d1d94f40e69076efe08615da9b_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:09,544 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/edf15ebbb8fe4efdbde36fa0c095006f, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:09,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/edf15ebbb8fe4efdbde36fa0c095006f is 175, key is test_row_0/A:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:09,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742380_1556 (size=30955) 2024-11-19T12:54:09,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020909761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020909761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020909761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020909761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020909771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:09,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/edf15ebbb8fe4efdbde36fa0c095006f 2024-11-19T12:54:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/e276d68cfd4b46e3a1c2907a67da35bc is 50, key is test_row_0/B:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:09,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742381_1557 (size=12001) 2024-11-19T12:54:09,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/e276d68cfd4b46e3a1c2907a67da35bc 2024-11-19T12:54:09,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/930944c35c62405fa9106bfab452f9c4 is 50, key is test_row_0/C:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:09,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742382_1558 (size=12001) 2024-11-19T12:54:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020910271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020910271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020910272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020910272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020910281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:10,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/930944c35c62405fa9106bfab452f9c4 2024-11-19T12:54:10,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/edf15ebbb8fe4efdbde36fa0c095006f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f 2024-11-19T12:54:10,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f, entries=150, sequenceid=100, filesize=30.2 K 2024-11-19T12:54:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/e276d68cfd4b46e3a1c2907a67da35bc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc 2024-11-19T12:54:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc, entries=150, sequenceid=100, filesize=11.7 K 2024-11-19T12:54:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/930944c35c62405fa9106bfab452f9c4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4 2024-11-19T12:54:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4, entries=150, sequenceid=100, filesize=11.7 K 2024-11-19T12:54:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1283ms, sequenceid=100, compaction requested=true 2024-11-19T12:54:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:10,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:54:10,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:10,397 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:10,397 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:10,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:54:10,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:10,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:54:10,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,398 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:10,398 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,398 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/B is initiating minor compaction (all files) 2024-11-19T12:54:10,398 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/A is initiating minor compaction (all files) 2024-11-19T12:54:10,398 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/B in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:10,398 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/A in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:10,399 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=44.6 K 2024-11-19T12:54:10,399 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=129.3 K 2024-11-19T12:54:10,399 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f] 2024-11-19T12:54:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa5bc48722241428b9519e55271b105, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732020846849 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79620c87ad3d4993b66d353605242b64, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732020846849 2024-11-19T12:54:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7079c4c24324ecc97fd372f7320b173, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732020846905 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting bf628a34eced42a997ff83d05f8f0284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732020846914 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a1883d0c12d544a8a6cbef5175cf4e54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732020847581 2024-11-19T12:54:10,399 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a7eddf24ed431984db5344980c3949, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732020847581 2024-11-19T12:54:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,400 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting e276d68cfd4b46e3a1c2907a67da35bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,400 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf15ebbb8fe4efdbde36fa0c095006f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,406 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,406 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/d0fadaa1b76145bfa0c9122ed73a6238 is 50, key is test_row_0/B:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,415 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,417 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119087ccc4b1d1f4e65863b3635a7f40f5c_9e9d5d81a38b9f7fdc417436276ba9a3 store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,419 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119087ccc4b1d1f4e65863b3635a7f40f5c_9e9d5d81a38b9f7fdc417436276ba9a3, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,420 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119087ccc4b1d1f4e65863b3635a7f40f5c_9e9d5d81a38b9f7fdc417436276ba9a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742383_1559 (size=12139) 2024-11-19T12:54:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,439 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/d0fadaa1b76145bfa0c9122ed73a6238 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/d0fadaa1b76145bfa0c9122ed73a6238 2024-11-19T12:54:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,443 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/B of 9e9d5d81a38b9f7fdc417436276ba9a3 into d0fadaa1b76145bfa0c9122ed73a6238(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:10,443 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:10,443 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/B, priority=12, startTime=1732020850397; duration=0sec 2024-11-19T12:54:10,443 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:10,443 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:B 2024-11-19T12:54:10,443 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,445 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:10,445 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/C is initiating minor compaction (all files) 2024-11-19T12:54:10,445 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/C in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:10,445 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=44.6 K 2024-11-19T12:54:10,445 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3c58624c8f4fab9ed2f54033a96023, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732020846849 2024-11-19T12:54:10,445 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbf1af348774eaf803d4d9089919035, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732020846914 2024-11-19T12:54:10,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f978960047469d93d959b03fb30527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732020847581 2024-11-19T12:54:10,446 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 930944c35c62405fa9106bfab452f9c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742384_1560 (size=4469) 2024-11-19T12:54:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,456 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#C#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:10,456 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/b8ea5c91de2248a888b0fc76fcc9a427 is 50, key is test_row_0/C:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742385_1561 (size=12139) 2024-11-19T12:54:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,853 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,853 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/1ed518f8da154874a3f19e5bc08c7a85 is 175, key is test_row_0/A:col10/1732020848784/Put/seqid=0 2024-11-19T12:54:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,881 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/b8ea5c91de2248a888b0fc76fcc9a427 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/b8ea5c91de2248a888b0fc76fcc9a427 2024-11-19T12:54:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,885 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/C of 9e9d5d81a38b9f7fdc417436276ba9a3 into b8ea5c91de2248a888b0fc76fcc9a427(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:10,885 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:10,885 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/C, priority=12, startTime=1732020850398; duration=0sec 2024-11-19T12:54:10,885 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:10,885 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:C 2024-11-19T12:54:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742386_1562 (size=31093) 2024-11-19T12:54:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,905 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/1ed518f8da154874a3f19e5bc08c7a85 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85 2024-11-19T12:54:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,911 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/A of 9e9d5d81a38b9f7fdc417436276ba9a3 into 1ed518f8da154874a3f19e5bc08c7a85(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:10,911 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,911 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/A, priority=12, startTime=1732020850397; duration=0sec 2024-11-19T12:54:10,911 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:10,911 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:A 2024-11-19T12:54:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T12:54:11,002 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T12:54:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:11,005 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:11,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T12:54:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:11,160 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939f76b0a5b8341508cff6e80c3f3cf5d_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020849115/Put/seqid=0 2024-11-19T12:54:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742387_1563 (size=9714) 2024-11-19T12:54:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:11,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020851322, max timestamp=1732020851328Num uniques 859; 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020851322, max timestamp=1732020851328Num uniques 859; 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020851322, max timestamp=1732020851328Num uniques 859; 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:11,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:11,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732020851338, max timestamp=1732020851356Num uniques 1018; 2024-11-19T12:54:11,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T12:54:11,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732020851338, max timestamp=1732020851356Num uniques 1018; 2024-11-19T12:54:11,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T12:54:11,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732020851352, max timestamp=1732020851356Num uniques 1018; 2024-11-19T12:54:11,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-19T12:54:11,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732020851370, max timestamp=1732020851381Num uniques 1177; 2024-11-19T12:54:11,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=17262, suffix heap size=150376,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=377 2024-11-19T12:54:11,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732020851370, max timestamp=1732020851381Num uniques 1009; 2024-11-19T12:54:11,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=17262, suffix heap size=132904,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=377 2024-11-19T12:54:11,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732020851370, max timestamp=1732020851381Num uniques 1177; 2024-11-19T12:54:11,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=17262, suffix heap size=150376,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=377 2024-11-19T12:54:11,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020911398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020911401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020911402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020911402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020911402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020911512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020911512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020911512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020911512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020911512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:11,605 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939f76b0a5b8341508cff6e80c3f3cf5d_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f76b0a5b8341508cff6e80c3f3cf5d_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/87ec436c6faa4c2c97bcc587947707da, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:11,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/87ec436c6faa4c2c97bcc587947707da is 175, key is test_row_0/A:col10/1732020849115/Put/seqid=0 2024-11-19T12:54:11,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742388_1564 (size=22361) 2024-11-19T12:54:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:11,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020911721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020911721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020911721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020911721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020911721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,022 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/87ec436c6faa4c2c97bcc587947707da 2024-11-19T12:54:12,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/8f0288c1fc654558af0feed0a96bf452 is 50, key is test_row_0/B:col10/1732020849115/Put/seqid=0 2024-11-19T12:54:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742389_1565 (size=9657) 2024-11-19T12:54:12,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020912031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020912031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020912031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020912031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020912031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:12,432 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/8f0288c1fc654558af0feed0a96bf452 2024-11-19T12:54:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/278fbb7ed66944d5a7e3b5205ddecc5f is 50, key is test_row_0/C:col10/1732020849115/Put/seqid=0 2024-11-19T12:54:12,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742390_1566 (size=9657) 2024-11-19T12:54:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020912541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020912541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020912542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020912542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020912542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:12,863 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/278fbb7ed66944d5a7e3b5205ddecc5f 2024-11-19T12:54:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/87ec436c6faa4c2c97bcc587947707da as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da 2024-11-19T12:54:12,879 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da, entries=100, sequenceid=110, filesize=21.8 K 2024-11-19T12:54:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/8f0288c1fc654558af0feed0a96bf452 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452 2024-11-19T12:54:12,887 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452, entries=100, sequenceid=110, filesize=9.4 K 2024-11-19T12:54:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/278fbb7ed66944d5a7e3b5205ddecc5f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f 2024-11-19T12:54:12,892 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f, entries=100, sequenceid=110, filesize=9.4 K 2024-11-19T12:54:12,893 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1733ms, sequenceid=110, compaction requested=false 2024-11-19T12:54:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T12:54:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T12:54:12,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T12:54:12,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-11-19T12:54:12,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8950 sec 2024-11-19T12:54:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T12:54:13,152 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:13,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T12:54:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:13,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:13,155 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:13,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:13,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=161.02 KB heapSize=444.23 KB 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:13,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198444d64982f64fdc9393355daf67a6a5_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020851401/Put/seqid=0 2024-11-19T12:54:13,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742391_1567 (size=12304) 2024-11-19T12:54:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:13,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020913556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020913560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020913561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020913561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020913561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,567 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,567 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020913672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020913672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020913672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:13,726 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198444d64982f64fdc9393355daf67a6a5_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198444d64982f64fdc9393355daf67a6a5_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:13,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/57f741c238554a69bda8b6cc6f32dae4, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/57f741c238554a69bda8b6cc6f32dae4 is 175, key is test_row_0/A:col10/1732020851401/Put/seqid=0 2024-11-19T12:54:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742392_1568 (size=31105) 2024-11-19T12:54:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020913881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020913881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020913882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,130 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/57f741c238554a69bda8b6cc6f32dae4 2024-11-19T12:54:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/523f1e78365946efb1a8d56d91622337 is 50, key is test_row_0/B:col10/1732020851401/Put/seqid=0 2024-11-19T12:54:14,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742393_1569 (size=12151) 2024-11-19T12:54:14,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T12:54:14,192 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T12:54:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020914191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020914191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020914191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:14,538 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/523f1e78365946efb1a8d56d91622337 2024-11-19T12:54:14,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 is 50, key is test_row_0/C:col10/1732020851401/Put/seqid=0 2024-11-19T12:54:14,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742394_1570 (size=12151) 2024-11-19T12:54:14,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020914701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020914701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020914701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:14,949 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 2024-11-19T12:54:14,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/57f741c238554a69bda8b6cc6f32dae4 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4 2024-11-19T12:54:14,956 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4, entries=150, sequenceid=181, filesize=30.4 K 2024-11-19T12:54:14,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/523f1e78365946efb1a8d56d91622337 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337 2024-11-19T12:54:14,965 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T12:54:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 2024-11-19T12:54:14,973 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T12:54:14,973 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~444.19 KB/454848, currentSize=33.54 KB/34350 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1665ms, sequenceid=181, compaction requested=true 2024-11-19T12:54:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T12:54:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T12:54:14,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T12:54:14,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-19T12:54:14,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8260 sec 2024-11-19T12:54:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T12:54:15,301 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:15,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T12:54:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:15,307 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:15,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:15,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:15,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T12:54:15,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:15,463 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc2bcb25613c49a9afaca65a84e68362_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020853559/Put/seqid=0 2024-11-19T12:54:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742395_1571 (size=12304) 2024-11-19T12:54:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:15,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020855590, max timestamp=1732020855659Num uniques 859; 2024-11-19T12:54:15,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020855590, max timestamp=1732020855659Num uniques 859; 2024-11-19T12:54:15,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732020855590, max timestamp=1732020855659Num uniques 859; 2024-11-19T12:54:15,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:54:15,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:54:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T12:54:15,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020915711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:15,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020915711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:15,717 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:15,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020915713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:15,718 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:15,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020855671, max timestamp=1732020855715Num uniques 1068; 2024-11-19T12:54:15,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020855671, max timestamp=1732020855715Num uniques 1068; 2024-11-19T12:54:15,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T12:54:15,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T12:54:15,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020855671, max timestamp=1732020855715Num uniques 1068; 2024-11-19T12:54:15,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T12:54:15,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020855772, max timestamp=1732020855787Num uniques 1077; 2024-11-19T12:54:15,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T12:54:15,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020855772, max timestamp=1732020855787Num uniques 1077; 2024-11-19T12:54:15,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T12:54:15,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732020855772, max timestamp=1732020855787Num uniques 1077; 2024-11-19T12:54:15,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-19T12:54:15,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020855798, max timestamp=1732020855837Num uniques 1036; 2024-11-19T12:54:15,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T12:54:15,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020855798, max timestamp=1732020855837Num uniques 1036; 2024-11-19T12:54:15,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T12:54:15,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732020855798, max timestamp=1732020855837Num uniques 1036; 2024-11-19T12:54:15,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T12:54:15,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:15,879 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc2bcb25613c49a9afaca65a84e68362_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc2bcb25613c49a9afaca65a84e68362_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/f839600940544c03899640ef95c932bf, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:15,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/f839600940544c03899640ef95c932bf is 175, key is test_row_0/A:col10/1732020853559/Put/seqid=0 2024-11-19T12:54:15,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020915877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:15,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:15,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020855873, max timestamp=1732020855881Num uniques 1095; 2024-11-19T12:54:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020915881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:15,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:54:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:15,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020855873, max timestamp=1732020855881Num uniques 1095; 2024-11-19T12:54:15,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:54:15,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020855873, max timestamp=1732020855881Num uniques 1095; 2024-11-19T12:54:15,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T12:54:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742396_1572 (size=31105) 2024-11-19T12:54:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:15,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732020855960, max timestamp=1732020855995Num uniques 1054; 2024-11-19T12:54:15,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-19T12:54:15,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732020855960, max timestamp=1732020855995Num uniques 1054; 2024-11-19T12:54:15,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-19T12:54:15,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732020855960, max timestamp=1732020855995Num uniques 1054; 2024-11-19T12:54:15,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-19T12:54:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020916069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020916070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020916072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,289 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/f839600940544c03899640ef95c932bf 2024-11-19T12:54:16,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/34c36521f9c94af68fcdf4d3e3e8a0fb is 50, key is test_row_0/B:col10/1732020853559/Put/seqid=0 2024-11-19T12:54:16,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742397_1573 (size=12151) 2024-11-19T12:54:16,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020916391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020916391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020916391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020916702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020916702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020916702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:16,730 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/34c36521f9c94af68fcdf4d3e3e8a0fb 2024-11-19T12:54:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1916cf697d504bb4b734dbead8e19a65 is 50, key is test_row_0/C:col10/1732020853559/Put/seqid=0 2024-11-19T12:54:16,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742398_1574 (size=12151) 2024-11-19T12:54:16,756 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1916cf697d504bb4b734dbead8e19a65 2024-11-19T12:54:16,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/f839600940544c03899640ef95c932bf as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf 2024-11-19T12:54:16,768 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf, entries=150, sequenceid=189, filesize=30.4 K 2024-11-19T12:54:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/34c36521f9c94af68fcdf4d3e3e8a0fb as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb 2024-11-19T12:54:16,775 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T12:54:16,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/1916cf697d504bb4b734dbead8e19a65 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65 2024-11-19T12:54:16,783 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T12:54:16,783 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1320ms, sequenceid=189, compaction requested=true 2024-11-19T12:54:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T12:54:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T12:54:16,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T12:54:16,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-19T12:54:16,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4810 sec 2024-11-19T12:54:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:17,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=154.31 KB heapSize=436.55 KB 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T12:54:17,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195b782b65b75f459aa66ec54456cce916_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:17,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020917218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742399_1575 (size=14994) 2024-11-19T12:54:17,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020917223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020917223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020917331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020917332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020917341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T12:54:17,452 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:17,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T12:54:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:17,454 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:17,454 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:17,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020917541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020917541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020917552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:17,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:17,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195b782b65b75f459aa66ec54456cce916_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195b782b65b75f459aa66ec54456cce916_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:17,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/59f46bfd901041c49548b3f80fbb7763, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:17,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/59f46bfd901041c49548b3f80fbb7763 is 175, key is test_row_0/A:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:17,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742400_1576 (size=39949) 2024-11-19T12:54:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020917731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020917732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,736 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_0' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:17,736 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020917851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020917851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020917861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:17,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:17,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/59f46bfd901041c49548b3f80fbb7763 2024-11-19T12:54:18,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/cc75ca333d98423a8f74433affbc1ae1 is 50, key is test_row_0/B:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:18,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742401_1577 (size=12301) 2024-11-19T12:54:18,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/cc75ca333d98423a8f74433affbc1ae1 2024-11-19T12:54:18,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/625946a0960f446a877a9f5da3b1300b is 50, key is test_row_0/C:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:18,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742402_1578 (size=12301) 2024-11-19T12:54:18,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020918361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:18,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:18,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:18,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:18,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020918371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:18,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020918382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:18,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/625946a0960f446a877a9f5da3b1300b 2024-11-19T12:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/59f46bfd901041c49548b3f80fbb7763 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763 2024-11-19T12:54:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763, entries=200, sequenceid=313, filesize=39.0 K 2024-11-19T12:54:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/cc75ca333d98423a8f74433affbc1ae1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1 2024-11-19T12:54:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T12:54:18,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/625946a0960f446a877a9f5da3b1300b as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b 2024-11-19T12:54:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T12:54:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.50 KB/446976, currentSize=26.84 KB/27480 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1301ms, sequenceid=313, compaction requested=true 2024-11-19T12:54:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:18,515 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:54:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T12:54:18,515 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T12:54:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,516 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155613 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T12:54:18,516 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58399 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T12:54:18,516 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/B is initiating minor compaction (all files) 2024-11-19T12:54:18,516 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/A is initiating minor compaction (all files) 2024-11-19T12:54:18,516 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/A in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,516 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/B in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,516 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=152.0 K 2024-11-19T12:54:18,516 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/d0fadaa1b76145bfa0c9122ed73a6238, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=57.0 K 2024-11-19T12:54:18,516 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,516 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763] 2024-11-19T12:54:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,517 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0fadaa1b76145bfa0c9122ed73a6238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:18,517 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed518f8da154874a3f19e5bc08c7a85, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:18,517 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0288c1fc654558af0feed0a96bf452, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020849115 2024-11-19T12:54:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,518 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523f1e78365946efb1a8d56d91622337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020851400 2024-11-19T12:54:18,518 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ec436c6faa4c2c97bcc587947707da, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020849115 2024-11-19T12:54:18,518 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c36521f9c94af68fcdf4d3e3e8a0fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732020853553 2024-11-19T12:54:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,518 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f741c238554a69bda8b6cc6f32dae4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020851400 2024-11-19T12:54:18,518 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc75ca333d98423a8f74433affbc1ae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856069 2024-11-19T12:54:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,519 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f839600940544c03899640ef95c932bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732020853553 2024-11-19T12:54:18,519 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f46bfd901041c49548b3f80fbb7763, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856030 2024-11-19T12:54:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,524 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,531 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,534 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c188e523dcdf43fb92fc1a0fb10433ab_9e9d5d81a38b9f7fdc417436276ba9a3 store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,535 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,535 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/6d01595ea9f943a485a43e4388cf3de1 is 50, key is test_row_0/B:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c188e523dcdf43fb92fc1a0fb10433ab_9e9d5d81a38b9f7fdc417436276ba9a3, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:18,536 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c188e523dcdf43fb92fc1a0fb10433ab_9e9d5d81a38b9f7fdc417436276ba9a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195c7753b82c0e457e975548adfb5af3a0_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020857221/Put/seqid=0 2024-11-19T12:54:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742404_1580 (size=4469) 2024-11-19T12:54:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742403_1579 (size=12609) 2024-11-19T12:54:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742405_1581 (size=9914) 2024-11-19T12:54:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,945 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,945 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/122a1db8afdc44ddb3d16800ed588827 is 175, key is test_row_0/A:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742406_1582 (size=31563) 2024-11-19T12:54:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,954 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/6d01595ea9f943a485a43e4388cf3de1 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/6d01595ea9f943a485a43e4388cf3de1 2024-11-19T12:54:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,958 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/B of 9e9d5d81a38b9f7fdc417436276ba9a3 into 6d01595ea9f943a485a43e4388cf3de1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:18,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:18,959 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/B, priority=11, startTime=1732020858515; duration=0sec 2024-11-19T12:54:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:18,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:B 2024-11-19T12:54:18,959 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T12:54:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,961 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58399 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T12:54:18,961 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/C is initiating minor compaction (all files) 2024-11-19T12:54:18,961 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/C in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:18,961 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/b8ea5c91de2248a888b0fc76fcc9a427, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=57.0 K 2024-11-19T12:54:18,962 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8ea5c91de2248a888b0fc76fcc9a427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732020848782 2024-11-19T12:54:18,962 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278fbb7ed66944d5a7e3b5205ddecc5f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732020849115 2024-11-19T12:54:18,962 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974d0989b1cd4e9f9ffa34cd5e53b7d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732020851400 2024-11-19T12:54:18,962 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1916cf697d504bb4b734dbead8e19a65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732020853553 2024-11-19T12:54:18,962 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 625946a0960f446a877a9f5da3b1300b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856069 2024-11-19T12:54:18,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,972 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195c7753b82c0e457e975548adfb5af3a0_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195c7753b82c0e457e975548adfb5af3a0_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3dab205f540e41939421dedb2baa113d, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3dab205f540e41939421dedb2baa113d is 175, key is test_row_0/A:col10/1732020857221/Put/seqid=0 2024-11-19T12:54:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,978 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,979 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/37d910826e67442ab4e15e4701c9ff08 is 50, key is test_row_0/C:col10/1732020856069/Put/seqid=0 2024-11-19T12:54:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742407_1583 (size=22561) 2024-11-19T12:54:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742408_1584 (size=12609) 2024-11-19T12:54:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,359 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/122a1db8afdc44ddb3d16800ed588827 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827 2024-11-19T12:54:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,364 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/A of 9e9d5d81a38b9f7fdc417436276ba9a3 into 122a1db8afdc44ddb3d16800ed588827(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:19,364 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:19,364 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/A, priority=11, startTime=1732020858515; duration=0sec 2024-11-19T12:54:19,364 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:19,364 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:A 2024-11-19T12:54:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:19,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:19,385 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3dab205f540e41939421dedb2baa113d 2024-11-19T12:54:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a8d67207600345d189d6110fdad5c51a is 50, key is test_row_0/B:col10/1732020857221/Put/seqid=0 2024-11-19T12:54:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,394 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/37d910826e67442ab4e15e4701c9ff08 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/37d910826e67442ab4e15e4701c9ff08 2024-11-19T12:54:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742409_1585 (size=9857) 2024-11-19T12:54:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,397 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/C of 9e9d5d81a38b9f7fdc417436276ba9a3 into 37d910826e67442ab4e15e4701c9ff08(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:19,397 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:19,397 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/C, priority=11, startTime=1732020858515; duration=0sec 2024-11-19T12:54:19,397 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:19,397 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:C 2024-11-19T12:54:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:19,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020859413, max timestamp=1732020859431Num uniques 859; 2024-11-19T12:54:19,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:19,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020859413, max timestamp=1732020859419Num uniques 859; 2024-11-19T12:54:19,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:19,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732020859413, max timestamp=1732020859419Num uniques 859; 2024-11-19T12:54:19,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020859477, max timestamp=1732020859485Num uniques 1018; 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020859477, max timestamp=1732020859485Num uniques 1018; 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732020859477, max timestamp=1732020859485Num uniques 1018; 2024-11-19T12:54:19,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T12:54:19,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020919510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:19,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:19,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020859503, max timestamp=1732020859512Num uniques 1077; 2024-11-19T12:54:19,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T12:54:19,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020859503, max timestamp=1732020859512Num uniques 1077; 2024-11-19T12:54:19,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T12:54:19,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732020859503, max timestamp=1732020859512Num uniques 1077; 2024-11-19T12:54:19,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T12:54:19,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732020859537, max timestamp=1732020859543Num uniques 1086; 2024-11-19T12:54:19,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T12:54:19,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732020859537, max timestamp=1732020859543Num uniques 1086; 2024-11-19T12:54:19,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T12:54:19,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732020859537, max timestamp=1732020859543Num uniques 1086; 2024-11-19T12:54:19,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T12:54:19,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020859573, max timestamp=1732020859580Num uniques 1045; 2024-11-19T12:54:19,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T12:54:19,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020859573, max timestamp=1732020859580Num uniques 1045; 2024-11-19T12:54:19,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732020859573, max timestamp=1732020859580Num uniques 1045; 2024-11-19T12:54:19,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T12:54:19,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T12:54:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020859606, max timestamp=1732020859621Num uniques 1054; 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020859606, max timestamp=1732020859621Num uniques 1054; 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732020859606, max timestamp=1732020859621Num uniques 1054; 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T12:54:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:19,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T12:54:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020919621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:19,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732020859635, max timestamp=1732020859664Num uniques 1013; 2024-11-19T12:54:19,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T12:54:19,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732020859635, max timestamp=1732020859664Num uniques 1013; 2024-11-19T12:54:19,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T12:54:19,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732020859635, max timestamp=1732020859664Num uniques 1013; 2024-11-19T12:54:19,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732020859715, max timestamp=1732020859726Num uniques 1022; 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732020859715, max timestamp=1732020859726Num uniques 1022; 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732020859715, max timestamp=1732020859726Num uniques 1022; 2024-11-19T12:54:19,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732020859756, max timestamp=1732020859764Num uniques 1031; 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732020859756, max timestamp=1732020859764Num uniques 1031; 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732020859756, max timestamp=1732020859764Num uniques 1031; 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-19T12:54:19,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-19T12:54:19,796 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a8d67207600345d189d6110fdad5c51a 2024-11-19T12:54:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/d20eaa0e979c4d3bbf0ca8345b8855c7 is 50, key is test_row_0/C:col10/1732020857221/Put/seqid=0 2024-11-19T12:54:19,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732020859781, max timestamp=1732020859801Num uniques 1090; 2024-11-19T12:54:19,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T12:54:19,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732020859781, max timestamp=1732020859801Num uniques 1090; 2024-11-19T12:54:19,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T12:54:19,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732020859781, max timestamp=1732020859801Num uniques 1090; 2024-11-19T12:54:19,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-19T12:54:19,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742410_1586 (size=9857) 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732020859825, max timestamp=1732020859838Num uniques 1099; 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732020859825, max timestamp=1732020859838Num uniques 1099; 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732020859825, max timestamp=1732020859838Num uniques 1049; 2024-11-19T12:54:19,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732020859866, max timestamp=1732020859872Num uniques 1058; 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732020859866, max timestamp=1732020859872Num uniques 1058; 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732020859866, max timestamp=1732020859872Num uniques 1058; 2024-11-19T12:54:19,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732020859887, max timestamp=1732020859902Num uniques 1017; 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732020859887, max timestamp=1732020859902Num uniques 1017; 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732020859887, max timestamp=1732020859902Num uniques 1017; 2024-11-19T12:54:19,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-19T12:54:19,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732020859918, max timestamp=1732020859929Num uniques 1076; 2024-11-19T12:54:19,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-19T12:54:19,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732020859918, max timestamp=1732020859929Num uniques 1076; 2024-11-19T12:54:19,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-19T12:54:19,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732020859918, max timestamp=1732020859929Num uniques 1026; 2024-11-19T12:54:19,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-19T12:54:19,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020919960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:19,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732020859948, max timestamp=1732020859963Num uniques 1085; 2024-11-19T12:54:19,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-19T12:54:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1732020859948, max timestamp=1732020859963Num uniques 1085; 2024-11-19T12:54:19,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-19T12:54:19,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1732020859948, max timestamp=1732020859963Num uniques 1085; 2024-11-19T12:54:19,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-11-19T12:54:20,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732020859994, max timestamp=1732020860003Num uniques 1044; 2024-11-19T12:54:20,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-19T12:54:20,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732020859994, max timestamp=1732020860003Num uniques 1094; 2024-11-19T12:54:20,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-19T12:54:20,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732020859994, max timestamp=1732020860003Num uniques 1044; 2024-11-19T12:54:20,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-11-19T12:54:20,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732020860026, max timestamp=1732020860037Num uniques 1103; 2024-11-19T12:54:20,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T12:54:20,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732020860026, max timestamp=1732020860037Num uniques 1103; 2024-11-19T12:54:20,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T12:54:20,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732020860026, max timestamp=1732020860037Num uniques 1103; 2024-11-19T12:54:20,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T12:54:20,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732020860071, max timestamp=1732020860078Num uniques 1062; 2024-11-19T12:54:20,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-19T12:54:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732020860071, max timestamp=1732020860078Num uniques 1062; 2024-11-19T12:54:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1732020860071, max timestamp=1732020860078Num uniques 1062; 2024-11-19T12:54:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-19T12:54:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732020860101, max timestamp=1732020860108Num uniques 1021; 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732020860101, max timestamp=1732020860108Num uniques 1021; 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1732020860101, max timestamp=1732020860108Num uniques 1021; 2024-11-19T12:54:20,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-19T12:54:20,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1732020860129, max timestamp=1732020860140Num uniques 1080; 2024-11-19T12:54:20,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=8240, suffix heap size=140288,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=180 2024-11-19T12:54:20,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1732020860129, max timestamp=1732020860140Num uniques 1080; 2024-11-19T12:54:20,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=8240, suffix heap size=140288,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=180 2024-11-19T12:54:20,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1732020860129, max timestamp=1732020860140Num uniques 1080; 2024-11-19T12:54:20,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=8240, suffix heap size=140288,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=180 2024-11-19T12:54:20,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1732020860169, max timestamp=1732020860177Num uniques 1039; 2024-11-19T12:54:20,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1732020860169, max timestamp=1732020860177Num uniques 1039; 2024-11-19T12:54:20,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-11-19T12:54:20,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=10944, suffix heap size=136024,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=239 2024-11-19T12:54:20,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732020860169, max timestamp=1732020860177Num uniques 1039; 2024-11-19T12:54:20,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/aba5a916dfea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=6870, suffix heap size=136024,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=150 2024-11-19T12:54:20,209 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/d20eaa0e979c4d3bbf0ca8345b8855c7 2024-11-19T12:54:20,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3dab205f540e41939421dedb2baa113d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d 2024-11-19T12:54:20,215 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d, entries=100, sequenceid=320, filesize=22.0 K 2024-11-19T12:54:20,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/a8d67207600345d189d6110fdad5c51a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a 2024-11-19T12:54:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a, entries=100, sequenceid=320, filesize=9.6 K 2024-11-19T12:54:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/d20eaa0e979c4d3bbf0ca8345b8855c7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7 2024-11-19T12:54:20,222 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7, entries=100, sequenceid=320, filesize=9.6 K 2024-11-19T12:54:20,223 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=69.82 KB/71496 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1699ms, sequenceid=320, compaction requested=false 2024-11-19T12:54:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T12:54:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:20,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=76.53 KB heapSize=209.07 KB 2024-11-19T12:54:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:20,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T12:54:20,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7700 sec 2024-11-19T12:54:20,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7750 sec 2024-11-19T12:54:20,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196470daa6b5554474b8198a04d447c49c_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:20,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742411_1587 (size=14994) 2024-11-19T12:54:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020920254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020920258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020920259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020920371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020920371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020920371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020920581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020920581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020920581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:20,639 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196470daa6b5554474b8198a04d447c49c_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196470daa6b5554474b8198a04d447c49c_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:20,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6ac08933184746dcb223a5d334b38cce, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:20,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6ac08933184746dcb223a5d334b38cce is 175, key is test_row_0/A:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742412_1588 (size=39949) 2024-11-19T12:54:20,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=694, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6ac08933184746dcb223a5d334b38cce 2024-11-19T12:54:20,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/51b77cfffb9f4db69659a7493fab1060 is 50, key is test_row_0/B:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:20,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742413_1589 (size=12301) 2024-11-19T12:54:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020920891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020920891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020920892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/51b77cfffb9f4db69659a7493fab1060 2024-11-19T12:54:21,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/4ade6bdee93f4e878c11cd080d7464fd is 50, key is test_row_0/C:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:21,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742414_1590 (size=12301) 2024-11-19T12:54:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020921401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020921401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020921402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.86 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/4ade6bdee93f4e878c11cd080d7464fd 2024-11-19T12:54:21,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6ac08933184746dcb223a5d334b38cce as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce 2024-11-19T12:54:21,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce, entries=200, sequenceid=694, filesize=39.0 K 2024-11-19T12:54:21,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/51b77cfffb9f4db69659a7493fab1060 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060 2024-11-19T12:54:21,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060, entries=150, sequenceid=694, filesize=12.0 K 2024-11-19T12:54:21,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/4ade6bdee93f4e878c11cd080d7464fd as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd 2024-11-19T12:54:21,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd, entries=150, sequenceid=694, filesize=12.0 K 2024-11-19T12:54:21,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.53 KB/78366, heapSize ~209.02 KB/214040, currentSize=107.34 KB/109920 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1250ms, sequenceid=694, compaction requested=true 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:21,473 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:54:21,473 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:54:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/B is initiating minor compaction (all files) 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/A is initiating minor compaction (all files) 2024-11-19T12:54:21,474 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/B in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,474 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/A in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,474 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=91.9 K 2024-11-19T12:54:21,474 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/6d01595ea9f943a485a43e4388cf3de1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=34.0 K 2024-11-19T12:54:21,474 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce] 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 122a1db8afdc44ddb3d16800ed588827, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856069 2024-11-19T12:54:21,474 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d01595ea9f943a485a43e4388cf3de1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856069 2024-11-19T12:54:21,475 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d67207600345d189d6110fdad5c51a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732020857220 2024-11-19T12:54:21,475 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dab205f540e41939421dedb2baa113d, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732020857220 2024-11-19T12:54:21,475 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b77cfffb9f4db69659a7493fab1060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860220 2024-11-19T12:54:21,475 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ac08933184746dcb223a5d334b38cce, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860185 2024-11-19T12:54:21,485 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:21,487 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:21,487 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/db9c7936471045bea012f21c99a4e8bc is 50, key is test_row_0/B:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:21,489 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e5557a71aa1c42febeb59aed1b620d89_9e9d5d81a38b9f7fdc417436276ba9a3 store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:21,491 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e5557a71aa1c42febeb59aed1b620d89_9e9d5d81a38b9f7fdc417436276ba9a3, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:21,491 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e5557a71aa1c42febeb59aed1b620d89_9e9d5d81a38b9f7fdc417436276ba9a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:21,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742415_1591 (size=12711) 2024-11-19T12:54:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742416_1592 (size=4469) 2024-11-19T12:54:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T12:54:21,631 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:21,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T12:54:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:21,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:21,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:21,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:21,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:21,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:21,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e6146d044f1d4b0495389752d1a55ad6_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020861762/Put/seqid=0 2024-11-19T12:54:21,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742417_1593 (size=14994) 2024-11-19T12:54:21,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:21,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e6146d044f1d4b0495389752d1a55ad6_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e6146d044f1d4b0495389752d1a55ad6_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:21,779 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3ce419500f55465eb28d35108bd986b8, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:21,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3ce419500f55465eb28d35108bd986b8 is 175, key is test_row_0/A:col10/1732020861762/Put/seqid=0 2024-11-19T12:54:21,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742418_1594 (size=39949) 2024-11-19T12:54:21,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:21,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:21,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020921785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020921788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020921891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,895 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/db9c7936471045bea012f21c99a4e8bc as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/db9c7936471045bea012f21c99a4e8bc 2024-11-19T12:54:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,900 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/B of 9e9d5d81a38b9f7fdc417436276ba9a3 into db9c7936471045bea012f21c99a4e8bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:21,900 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:21,900 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:21,900 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/B, priority=13, startTime=1732020861473; duration=0sec 2024-11-19T12:54:21,900 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:21,900 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:B 2024-11-19T12:54:21,900 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T12:54:21,900 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/0daaa31ed5a845dbb28c9bc7bcd0a3df is 175, key is test_row_0/A:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:21,902 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T12:54:21,902 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/C is initiating minor compaction (all files) 2024-11-19T12:54:21,902 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/C in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,902 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/37d910826e67442ab4e15e4701c9ff08, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=34.0 K 2024-11-19T12:54:21,902 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d910826e67442ab4e15e4701c9ff08, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732020856069 2024-11-19T12:54:21,902 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting d20eaa0e979c4d3bbf0ca8345b8855c7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732020857220 2024-11-19T12:54:21,903 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ade6bdee93f4e878c11cd080d7464fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860220 2024-11-19T12:54:21,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742419_1595 (size=31665) 2024-11-19T12:54:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020921901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:21,909 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:21,910 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/68825979f3304845892bf87b569bb61a is 50, key is test_row_0/C:col10/1732020860220/Put/seqid=0 2024-11-19T12:54:21,910 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/0daaa31ed5a845dbb28c9bc7bcd0a3df as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df 2024-11-19T12:54:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742420_1596 (size=12711) 2024-11-19T12:54:21,916 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/A of 9e9d5d81a38b9f7fdc417436276ba9a3 into 0daaa31ed5a845dbb28c9bc7bcd0a3df(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:21,916 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:21,916 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/A, priority=13, startTime=1732020861473; duration=0sec 2024-11-19T12:54:21,916 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:21,916 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:A 2024-11-19T12:54:21,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:21,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:22,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020922101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020922111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=714, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3ce419500f55465eb28d35108bd986b8 2024-11-19T12:54:22,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/12e55a6685a44653b784ded717439907 is 50, key is test_row_0/B:col10/1732020861762/Put/seqid=0 2024-11-19T12:54:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742421_1597 (size=12301) 2024-11-19T12:54:22,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:22,317 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/68825979f3304845892bf87b569bb61a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/68825979f3304845892bf87b569bb61a 2024-11-19T12:54:22,321 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/C of 9e9d5d81a38b9f7fdc417436276ba9a3 into 68825979f3304845892bf87b569bb61a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:22,321 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:22,321 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/C, priority=13, startTime=1732020861473; duration=0sec 2024-11-19T12:54:22,321 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:22,321 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:C 2024-11-19T12:54:22,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020922411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020922421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020922421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,423 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,423 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020922421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020922421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,424 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/12e55a6685a44653b784ded717439907 2024-11-19T12:54:22,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/0b569ebe7ff54bbea0ea860317c5d384 is 50, key is test_row_0/C:col10/1732020861762/Put/seqid=0 2024-11-19T12:54:22,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742422_1598 (size=12301) 2024-11-19T12:54:22,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:22,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:22,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020922921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020922931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:23,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:23,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:23,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/0b569ebe7ff54bbea0ea860317c5d384 2024-11-19T12:54:23,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/3ce419500f55465eb28d35108bd986b8 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8 2024-11-19T12:54:23,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8, entries=200, sequenceid=714, filesize=39.0 K 2024-11-19T12:54:23,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/12e55a6685a44653b784ded717439907 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907 2024-11-19T12:54:23,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T12:54:23,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/0b569ebe7ff54bbea0ea860317c5d384 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384 2024-11-19T12:54:23,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T12:54:23,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1257ms, sequenceid=714, compaction requested=false 2024-11-19T12:54:23,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:23,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T12:54:23,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:23,159 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194ff6aba956a84bf28f181d9b823c475f_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020861783/Put/seqid=0 2024-11-19T12:54:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742423_1599 (size=12454) 2024-11-19T12:54:23,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:23,573 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194ff6aba956a84bf28f181d9b823c475f_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194ff6aba956a84bf28f181d9b823c475f_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:23,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6432f86e13e84bbcaec6b50437055897, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:23,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6432f86e13e84bbcaec6b50437055897 is 175, key is test_row_0/A:col10/1732020861783/Put/seqid=0 2024-11-19T12:54:23,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742424_1600 (size=31255) 2024-11-19T12:54:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:23,978 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=729, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6432f86e13e84bbcaec6b50437055897 2024-11-19T12:54:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020923975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020923975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/716527f98a524a348a51d99508f90cc9 is 50, key is test_row_0/B:col10/1732020861783/Put/seqid=0 2024-11-19T12:54:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742425_1601 (size=12301) 2024-11-19T12:54:24,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020924091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020924091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020924301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020924301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,390 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=729 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/716527f98a524a348a51d99508f90cc9 2024-11-19T12:54:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/3335b338bb6d4879b62acbe3f27e7652 is 50, key is test_row_0/C:col10/1732020861783/Put/seqid=0 2024-11-19T12:54:24,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742426_1602 (size=12301) 2024-11-19T12:54:24,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49336 deadline: 1732020924431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,433 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49354 deadline: 1732020924451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49368 deadline: 1732020924451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,453 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,454 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to aba5a916dfea,35349,1732020694570 for 'test_row_2' in 9e9d5d81a38b9f7fdc417436276ba9a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020924612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020924612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:24,799 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=729 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/3335b338bb6d4879b62acbe3f27e7652 2024-11-19T12:54:24,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/6432f86e13e84bbcaec6b50437055897 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897 2024-11-19T12:54:24,805 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897, entries=150, sequenceid=729, filesize=30.5 K 2024-11-19T12:54:24,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/716527f98a524a348a51d99508f90cc9 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9 2024-11-19T12:54:24,811 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9, entries=150, sequenceid=729, filesize=12.0 K 2024-11-19T12:54:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/3335b338bb6d4879b62acbe3f27e7652 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652 2024-11-19T12:54:24,814 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652, entries=150, sequenceid=729, filesize=12.0 K 2024-11-19T12:54:24,815 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1656ms, sequenceid=729, compaction requested=true 2024-11-19T12:54:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T12:54:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T12:54:24,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T12:54:24,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1820 sec 2024-11-19T12:54:24,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.1860 sec 2024-11-19T12:54:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:25,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T12:54:25,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192c71dd9a813c4e1c831472713c24c3d1_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742427_1603 (size=14994) 2024-11-19T12:54:25,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:25,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192c71dd9a813c4e1c831472713c24c3d1_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c71dd9a813c4e1c831472713c24c3d1_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:25,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b15cedaf193a4a19952910c104900b33, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:25,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b15cedaf193a4a19952910c104900b33 is 175, key is test_row_0/A:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:25,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742428_1604 (size=39949) 2024-11-19T12:54:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020925141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020925146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020925251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020925251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020925461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020925462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=751, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b15cedaf193a4a19952910c104900b33 2024-11-19T12:54:25,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/f94eabb60c774ce4942ec10abcbbf655 is 50, key is test_row_0/B:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:25,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742429_1605 (size=12301) 2024-11-19T12:54:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020925771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020925771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T12:54:25,802 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:25,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T12:54:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T12:54:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:25,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T12:54:25,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T12:54:25,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T12:54:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:25,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T12:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:25,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:25,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=751 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/f94eabb60c774ce4942ec10abcbbf655 2024-11-19T12:54:25,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/33c912511bcb4a9bb1d6f3342b9dc973 is 50, key is test_row_0/C:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742430_1606 (size=12301) 2024-11-19T12:54:26,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T12:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:26,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T12:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49348 deadline: 1732020926281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:26,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T12:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35349 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49396 deadline: 1732020926281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 , the old value is region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T12:54:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., hostname=aba5a916dfea,35349,1732020694570, seqNum=5 because the exception is null or not the one we care about 2024-11-19T12:54:26,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=751 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/33c912511bcb4a9bb1d6f3342b9dc973 2024-11-19T12:54:26,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/b15cedaf193a4a19952910c104900b33 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33 2024-11-19T12:54:26,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33, entries=200, sequenceid=751, filesize=39.0 K 2024-11-19T12:54:26,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/f94eabb60c774ce4942ec10abcbbf655 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655 2024-11-19T12:54:26,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655, entries=150, sequenceid=751, filesize=12.0 K 2024-11-19T12:54:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/33c912511bcb4a9bb1d6f3342b9dc973 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973 2024-11-19T12:54:26,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973, entries=150, sequenceid=751, filesize=12.0 K 2024-11-19T12:54:26,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1265ms, sequenceid=751, compaction requested=true 2024-11-19T12:54:26,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:26,389 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T12:54:26,390 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e9d5d81a38b9f7fdc417436276ba9a3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T12:54:26,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:26,390 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:26,390 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/A is initiating minor compaction (all files) 2024-11-19T12:54:26,390 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:26,390 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/B is initiating minor compaction (all files) 2024-11-19T12:54:26,390 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/A in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,390 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/B in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,390 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/db9c7936471045bea012f21c99a4e8bc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=48.5 K 2024-11-19T12:54:26,390 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=139.5 K 2024-11-19T12:54:26,390 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. files: [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33] 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting db9c7936471045bea012f21c99a4e8bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860220 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0daaa31ed5a845dbb28c9bc7bcd0a3df, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860220 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e55a6685a44653b784ded717439907, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1732020860253 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce419500f55465eb28d35108bd986b8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1732020860253 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 716527f98a524a348a51d99508f90cc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=729, earliestPutTs=1732020861775 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6432f86e13e84bbcaec6b50437055897, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=729, earliestPutTs=1732020861775 2024-11-19T12:54:26,391 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting f94eabb60c774ce4942ec10abcbbf655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=751, earliestPutTs=1732020863973 2024-11-19T12:54:26,392 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b15cedaf193a4a19952910c104900b33, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=751, earliestPutTs=1732020863973 2024-11-19T12:54:26,397 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:26,398 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#B#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:26,398 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/5ca56a379770409784ecd9004632127d is 50, key is test_row_0/B:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:26,399 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c07d42a5c2ca472189de37ba30046a84_9e9d5d81a38b9f7fdc417436276ba9a3 store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:26,401 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c07d42a5c2ca472189de37ba30046a84_9e9d5d81a38b9f7fdc417436276ba9a3, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:26,401 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c07d42a5c2ca472189de37ba30046a84_9e9d5d81a38b9f7fdc417436276ba9a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:26,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742432_1608 (size=4469) 2024-11-19T12:54:26,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742431_1607 (size=12847) 2024-11-19T12:54:26,412 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#A#compaction#516 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:26,412 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a5d26299a739448c9725ab7f70658501 is 175, key is test_row_0/A:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:26,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,413 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:26,416 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/5ca56a379770409784ecd9004632127d as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5ca56a379770409784ecd9004632127d 2024-11-19T12:54:26,423 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/B of 9e9d5d81a38b9f7fdc417436276ba9a3 into 5ca56a379770409784ecd9004632127d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:26,423 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:26,423 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/B, priority=12, startTime=1732020866390; duration=0sec 2024-11-19T12:54:26,423 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T12:54:26,423 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:B 2024-11-19T12:54:26,423 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T12:54:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742433_1609 (size=31801) 2024-11-19T12:54:26,424 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T12:54:26,425 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1541): 9e9d5d81a38b9f7fdc417436276ba9a3/C is initiating minor compaction (all files) 2024-11-19T12:54:26,425 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e9d5d81a38b9f7fdc417436276ba9a3/C in TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:26,425 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/68825979f3304845892bf87b569bb61a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973] into tmpdir=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp, totalSize=48.5 K 2024-11-19T12:54:26,425 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 68825979f3304845892bf87b569bb61a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1732020860220 2024-11-19T12:54:26,425 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b569ebe7ff54bbea0ea860317c5d384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1732020860253 2024-11-19T12:54:26,426 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 3335b338bb6d4879b62acbe3f27e7652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=729, earliestPutTs=1732020861775 2024-11-19T12:54:26,427 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c912511bcb4a9bb1d6f3342b9dc973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=751, earliestPutTs=1732020863973 2024-11-19T12:54:26,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e5aa6ceefba4449c9979d1844c49deb7_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_0/A:col10/1732020865132/Put/seqid=0 2024-11-19T12:54:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:26,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742434_1610 (size=12454) 2024-11-19T12:54:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:26,437 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e9d5d81a38b9f7fdc417436276ba9a3#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T12:54:26,438 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/771aa1e3b9d14b8cb8d004f10caac6f5 is 50, key is test_row_0/C:col10/1732020863974/Put/seqid=0 2024-11-19T12:54:26,441 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e5aa6ceefba4449c9979d1844c49deb7_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5aa6ceefba4449c9979d1844c49deb7_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:26,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a2ffc8f5c6864e14aff8924220600a14, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a2ffc8f5c6864e14aff8924220600a14 is 175, key is test_row_0/A:col10/1732020865132/Put/seqid=0 2024-11-19T12:54:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742435_1611 (size=12847) 2024-11-19T12:54:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742436_1612 (size=31255) 2024-11-19T12:54:26,828 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a5d26299a739448c9725ab7f70658501 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a5d26299a739448c9725ab7f70658501 2024-11-19T12:54:26,832 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/A of 9e9d5d81a38b9f7fdc417436276ba9a3 into a5d26299a739448c9725ab7f70658501(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:26,832 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:26,832 INFO [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/A, priority=12, startTime=1732020866389; duration=0sec 2024-11-19T12:54:26,832 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:26,832 DEBUG [RS:0;aba5a916dfea:35349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:A 2024-11-19T12:54:26,846 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=761, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a2ffc8f5c6864e14aff8924220600a14 2024-11-19T12:54:26,847 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-19T12:54:26,847 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:26,847 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,848 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:26,849 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-19T12:54:26,849 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-19T12:54:26,849 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:26,849 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:26,849 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,849 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,849 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,849 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:26,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:26,849 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-19T12:54:26,849 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:26,849 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,850 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:26,850 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-11-19T12:54:26,850 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:26,850 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,850 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:26,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:26,851 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/771aa1e3b9d14b8cb8d004f10caac6f5 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/771aa1e3b9d14b8cb8d004f10caac6f5 2024-11-19T12:54:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/86dfbabddec6410691fe4e9a14098ef0 is 50, key is test_row_0/B:col10/1732020865132/Put/seqid=0 2024-11-19T12:54:26,854 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e9d5d81a38b9f7fdc417436276ba9a3/C of 9e9d5d81a38b9f7fdc417436276ba9a3 into 771aa1e3b9d14b8cb8d004f10caac6f5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T12:54:26,854 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:26,854 INFO [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., storeName=9e9d5d81a38b9f7fdc417436276ba9a3/C, priority=12, startTime=1732020866390; duration=0sec 2024-11-19T12:54:26,854 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T12:54:26,854 DEBUG [RS:0;aba5a916dfea:35349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e9d5d81a38b9f7fdc417436276ba9a3:C 2024-11-19T12:54:26,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742437_1613 (size=12301) 2024-11-19T12:54:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:27,256 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/86dfbabddec6410691fe4e9a14098ef0 2024-11-19T12:54:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/a338824cc2dd496a896e1681c7d29b10 is 50, key is test_row_0/C:col10/1732020865132/Put/seqid=0 2024-11-19T12:54:27,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742438_1614 (size=12301) 2024-11-19T12:54:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35349 {}] regionserver.HRegion(8855): Flush requested on 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:27,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. as already flushing 2024-11-19T12:54:27,294 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-19T12:54:27,294 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:27,294 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:27,295 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:27,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:27,303 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-19T12:54:27,303 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:27,303 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:27,303 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:27,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:27,663 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/a338824cc2dd496a896e1681c7d29b10 2024-11-19T12:54:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/a2ffc8f5c6864e14aff8924220600a14 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a2ffc8f5c6864e14aff8924220600a14 2024-11-19T12:54:27,671 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a2ffc8f5c6864e14aff8924220600a14, entries=150, sequenceid=761, filesize=30.5 K 2024-11-19T12:54:27,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/86dfbabddec6410691fe4e9a14098ef0 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/86dfbabddec6410691fe4e9a14098ef0 2024-11-19T12:54:27,675 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/86dfbabddec6410691fe4e9a14098ef0, entries=150, sequenceid=761, filesize=12.0 K 2024-11-19T12:54:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/a338824cc2dd496a896e1681c7d29b10 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/a338824cc2dd496a896e1681c7d29b10 2024-11-19T12:54:27,678 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/a338824cc2dd496a896e1681c7d29b10, entries=150, sequenceid=761, filesize=12.0 K 2024-11-19T12:54:27,679 INFO [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1266ms, sequenceid=761, compaction requested=false 2024-11-19T12:54:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 9e9d5d81a38b9f7fdc417436276ba9a3: 2024-11-19T12:54:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T12:54:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T12:54:27,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T12:54:27,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8770 sec 2024-11-19T12:54:27,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8810 sec 2024-11-19T12:54:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T12:54:27,952 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:28,456 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-19T12:54:28,456 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:28,456 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,457 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:28,473 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-19T12:54:28,473 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:28,474 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,474 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:28,500 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-19T12:54:28,500 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T12:54:28,500 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,500 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 232 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2990 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3035 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2928 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2996 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3051 2024-11-19T12:54:28,500 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T12:54:28,500 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:28,500 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:28,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:28,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:28,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T12:54:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:28,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020868503"}]},"ts":"1732020868503"} 2024-11-19T12:54:28,504 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T12:54:28,504 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T12:54:28,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T12:54:28,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, UNASSIGN}] 2024-11-19T12:54:28,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, UNASSIGN 2024-11-19T12:54:28,507 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=CLOSING, regionLocation=aba5a916dfea,35349,1732020694570 2024-11-19T12:54:28,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, UNASSIGN because future has completed 2024-11-19T12:54:28,508 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:54:28,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570}] 2024-11-19T12:54:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:28,660 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:28,660 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:54:28,660 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 9e9d5d81a38b9f7fdc417436276ba9a3, disabling compactions & flushes 2024-11-19T12:54:28,660 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:28,660 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:28,660 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. after waiting 0 ms 2024-11-19T12:54:28,660 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:28,661 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 9e9d5d81a38b9f7fdc417436276ba9a3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=A 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=B 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9e9d5d81a38b9f7fdc417436276ba9a3, store=C 2024-11-19T12:54:28,661 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T12:54:28,669 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191eade730566a4c75acb27fae2d2d5e74_9e9d5d81a38b9f7fdc417436276ba9a3 is 50, key is test_row_1/A:col10/1732020867302/Put/seqid=0 2024-11-19T12:54:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742439_1615 (size=9914) 2024-11-19T12:54:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:29,073 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:29,079 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191eade730566a4c75acb27fae2d2d5e74_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191eade730566a4c75acb27fae2d2d5e74_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:29,081 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/9cbcf1f8995f4f5ba81e8b42d091c2b7, store: [table=TestAcidGuarantees family=A region=9e9d5d81a38b9f7fdc417436276ba9a3] 2024-11-19T12:54:29,082 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 is 175, key is test_row_1/A:col10/1732020867302/Put/seqid=0 2024-11-19T12:54:29,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742440_1616 (size=22561) 2024-11-19T12:54:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:29,489 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=772, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 2024-11-19T12:54:29,498 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/54891e37dabe40b293423ece2ad3ef9f is 50, key is test_row_1/B:col10/1732020867302/Put/seqid=0 2024-11-19T12:54:29,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742441_1617 (size=9857) 2024-11-19T12:54:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:29,902 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=772 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/54891e37dabe40b293423ece2ad3ef9f 2024-11-19T12:54:29,915 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/50b4dadd17134ae19699c8ee965620fa is 50, key is test_row_1/C:col10/1732020867302/Put/seqid=0 2024-11-19T12:54:29,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742442_1618 (size=9857) 2024-11-19T12:54:30,319 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=772 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/50b4dadd17134ae19699c8ee965620fa 2024-11-19T12:54:30,323 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 2024-11-19T12:54:30,325 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/9cbcf1f8995f4f5ba81e8b42d091c2b7, entries=100, sequenceid=772, filesize=22.0 K 2024-11-19T12:54:30,326 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/B/54891e37dabe40b293423ece2ad3ef9f as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/54891e37dabe40b293423ece2ad3ef9f 2024-11-19T12:54:30,328 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/54891e37dabe40b293423ece2ad3ef9f, entries=100, sequenceid=772, filesize=9.6 K 2024-11-19T12:54:30,329 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/.tmp/C/50b4dadd17134ae19699c8ee965620fa as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/50b4dadd17134ae19699c8ee965620fa 2024-11-19T12:54:30,331 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/50b4dadd17134ae19699c8ee965620fa, entries=100, sequenceid=772, filesize=9.6 K 2024-11-19T12:54:30,332 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1671ms, sequenceid=772, compaction requested=true 2024-11-19T12:54:30,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33] to archive 2024-11-19T12:54:30,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:30,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/79620c87ad3d4993b66d353605242b64 2024-11-19T12:54:30,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b7079c4c24324ecc97fd372f7320b173 2024-11-19T12:54:30,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b0a7eddf24ed431984db5344980c3949 2024-11-19T12:54:30,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/1ed518f8da154874a3f19e5bc08c7a85 2024-11-19T12:54:30,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/edf15ebbb8fe4efdbde36fa0c095006f 2024-11-19T12:54:30,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/87ec436c6faa4c2c97bcc587947707da 2024-11-19T12:54:30,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/57f741c238554a69bda8b6cc6f32dae4 2024-11-19T12:54:30,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/f839600940544c03899640ef95c932bf 2024-11-19T12:54:30,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/59f46bfd901041c49548b3f80fbb7763 2024-11-19T12:54:30,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/122a1db8afdc44ddb3d16800ed588827 2024-11-19T12:54:30,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3dab205f540e41939421dedb2baa113d 2024-11-19T12:54:30,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6ac08933184746dcb223a5d334b38cce 2024-11-19T12:54:30,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/0daaa31ed5a845dbb28c9bc7bcd0a3df 2024-11-19T12:54:30,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/3ce419500f55465eb28d35108bd986b8 2024-11-19T12:54:30,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/6432f86e13e84bbcaec6b50437055897 2024-11-19T12:54:30,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/b15cedaf193a4a19952910c104900b33 2024-11-19T12:54:30,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/d0fadaa1b76145bfa0c9122ed73a6238, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/6d01595ea9f943a485a43e4388cf3de1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/db9c7936471045bea012f21c99a4e8bc, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655] to archive 2024-11-19T12:54:30,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:30,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5aa5bc48722241428b9519e55271b105 2024-11-19T12:54:30,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/bf628a34eced42a997ff83d05f8f0284 2024-11-19T12:54:30,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a1883d0c12d544a8a6cbef5175cf4e54 2024-11-19T12:54:30,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/d0fadaa1b76145bfa0c9122ed73a6238 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/d0fadaa1b76145bfa0c9122ed73a6238 2024-11-19T12:54:30,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/e276d68cfd4b46e3a1c2907a67da35bc 2024-11-19T12:54:30,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/8f0288c1fc654558af0feed0a96bf452 2024-11-19T12:54:30,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/523f1e78365946efb1a8d56d91622337 2024-11-19T12:54:30,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/34c36521f9c94af68fcdf4d3e3e8a0fb 2024-11-19T12:54:30,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/6d01595ea9f943a485a43e4388cf3de1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/6d01595ea9f943a485a43e4388cf3de1 2024-11-19T12:54:30,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/cc75ca333d98423a8f74433affbc1ae1 2024-11-19T12:54:30,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/a8d67207600345d189d6110fdad5c51a 2024-11-19T12:54:30,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/db9c7936471045bea012f21c99a4e8bc to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/db9c7936471045bea012f21c99a4e8bc 2024-11-19T12:54:30,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/51b77cfffb9f4db69659a7493fab1060 2024-11-19T12:54:30,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/12e55a6685a44653b784ded717439907 2024-11-19T12:54:30,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/716527f98a524a348a51d99508f90cc9 2024-11-19T12:54:30,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/f94eabb60c774ce4942ec10abcbbf655 2024-11-19T12:54:30,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/b8ea5c91de2248a888b0fc76fcc9a427, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/37d910826e67442ab4e15e4701c9ff08, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/68825979f3304845892bf87b569bb61a, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973] to archive 2024-11-19T12:54:30,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T12:54:30,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/7d3c58624c8f4fab9ed2f54033a96023 2024-11-19T12:54:30,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1bbf1af348774eaf803d4d9089919035 2024-11-19T12:54:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/26f978960047469d93d959b03fb30527 2024-11-19T12:54:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/b8ea5c91de2248a888b0fc76fcc9a427 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/b8ea5c91de2248a888b0fc76fcc9a427 2024-11-19T12:54:30,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/930944c35c62405fa9106bfab452f9c4 2024-11-19T12:54:30,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/278fbb7ed66944d5a7e3b5205ddecc5f 2024-11-19T12:54:30,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/974d0989b1cd4e9f9ffa34cd5e53b7d5 2024-11-19T12:54:30,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/1916cf697d504bb4b734dbead8e19a65 2024-11-19T12:54:30,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/37d910826e67442ab4e15e4701c9ff08 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/37d910826e67442ab4e15e4701c9ff08 2024-11-19T12:54:30,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/625946a0960f446a877a9f5da3b1300b 2024-11-19T12:54:30,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/d20eaa0e979c4d3bbf0ca8345b8855c7 2024-11-19T12:54:30,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/68825979f3304845892bf87b569bb61a to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/68825979f3304845892bf87b569bb61a 2024-11-19T12:54:30,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/4ade6bdee93f4e878c11cd080d7464fd 2024-11-19T12:54:30,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/0b569ebe7ff54bbea0ea860317c5d384 2024-11-19T12:54:30,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/3335b338bb6d4879b62acbe3f27e7652 2024-11-19T12:54:30,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/33c912511bcb4a9bb1d6f3342b9dc973 2024-11-19T12:54:30,376 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits/775.seqid, newMaxSeqId=775, maxSeqId=4 2024-11-19T12:54:30,376 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. 2024-11-19T12:54:30,376 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 9e9d5d81a38b9f7fdc417436276ba9a3: Waiting for close lock at 1732020868660Running coprocessor pre-close hooks at 1732020868660Disabling compacts and flushes for region at 1732020868660Disabling writes for close at 1732020868660Obtaining lock to block concurrent updates at 1732020868661 (+1 ms)Preparing flush snapshotting stores in 9e9d5d81a38b9f7fdc417436276ba9a3 at 1732020868661Finished memstore snapshotting TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732020868661Flushing stores of TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3. at 1732020868662 (+1 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/A: creating writer at 1732020868662Flushing mob file 9e9d5d81a38b9f7fdc417436276ba9a3/A: appending metadata at 1732020868669 (+7 ms)Flushing mob file 9e9d5d81a38b9f7fdc417436276ba9a3/A: closing flushed file at 1732020868669Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/A: appending metadata at 1732020869081 (+412 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/A: closing flushed file at 1732020869081Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/B: creating writer at 1732020869494 (+413 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/B: appending metadata at 1732020869497 (+3 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/B: closing flushed file at 1732020869497Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/C: creating writer at 1732020869909 (+412 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/C: appending metadata at 1732020869914 (+5 ms)Flushing 9e9d5d81a38b9f7fdc417436276ba9a3/C: closing flushed file at 1732020869914Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5179d5f2: reopening flushed file at 1732020870323 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cb4224b: reopening flushed file at 1732020870325 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ff6e194: reopening flushed file at 1732020870328 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9e9d5d81a38b9f7fdc417436276ba9a3 in 1671ms, sequenceid=772, compaction requested=true at 1732020870332 (+4 ms)Writing region close event to WAL at 1732020870374 (+42 ms)Running coprocessor post-close hooks at 1732020870376 (+2 ms)Closed at 1732020870376 2024-11-19T12:54:30,378 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,378 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=9e9d5d81a38b9f7fdc417436276ba9a3, regionState=CLOSED 2024-11-19T12:54:30,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 because future has completed 2024-11-19T12:54:30,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T12:54:30,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e9d5d81a38b9f7fdc417436276ba9a3, server=aba5a916dfea,35349,1732020694570 in 1.8710 sec 2024-11-19T12:54:30,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-19T12:54:30,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e9d5d81a38b9f7fdc417436276ba9a3, UNASSIGN in 1.8740 sec 2024-11-19T12:54:30,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T12:54:30,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-19T12:54:30,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020870384"}]},"ts":"1732020870384"} 2024-11-19T12:54:30,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T12:54:30,385 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T12:54:30,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8840 sec 2024-11-19T12:54:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T12:54:30,652 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:30,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T12:54:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,655 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T12:54:30,657 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,661 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,665 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C, FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits] 2024-11-19T12:54:30,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/9cbcf1f8995f4f5ba81e8b42d091c2b7 2024-11-19T12:54:30,670 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a2ffc8f5c6864e14aff8924220600a14 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a2ffc8f5c6864e14aff8924220600a14 2024-11-19T12:54:30,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a5d26299a739448c9725ab7f70658501 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/A/a5d26299a739448c9725ab7f70658501 2024-11-19T12:54:30,675 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/54891e37dabe40b293423ece2ad3ef9f to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/54891e37dabe40b293423ece2ad3ef9f 2024-11-19T12:54:30,677 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5ca56a379770409784ecd9004632127d to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/5ca56a379770409784ecd9004632127d 2024-11-19T12:54:30,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/86dfbabddec6410691fe4e9a14098ef0 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/B/86dfbabddec6410691fe4e9a14098ef0 2024-11-19T12:54:30,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/50b4dadd17134ae19699c8ee965620fa to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/50b4dadd17134ae19699c8ee965620fa 2024-11-19T12:54:30,683 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/771aa1e3b9d14b8cb8d004f10caac6f5 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/771aa1e3b9d14b8cb8d004f10caac6f5 2024-11-19T12:54:30,684 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/a338824cc2dd496a896e1681c7d29b10 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/C/a338824cc2dd496a896e1681c7d29b10 2024-11-19T12:54:30,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits/775.seqid to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3/recovered.edits/775.seqid 2024-11-19T12:54:30,689 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/default/TestAcidGuarantees/9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,689 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T12:54:30,689 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:54:30,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T12:54:30,694 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191eade730566a4c75acb27fae2d2d5e74_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191eade730566a4c75acb27fae2d2d5e74_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,696 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c71dd9a813c4e1c831472713c24c3d1_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c71dd9a813c4e1c831472713c24c3d1_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,698 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f76b0a5b8341508cff6e80c3f3cf5d_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939f76b0a5b8341508cff6e80c3f3cf5d_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,700 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cd270f201de47ce8dca843abbf65137_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cd270f201de47ce8dca843abbf65137_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194ff6aba956a84bf28f181d9b823c475f_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194ff6aba956a84bf28f181d9b823c475f_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,705 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195b782b65b75f459aa66ec54456cce916_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195b782b65b75f459aa66ec54456cce916_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,707 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195c7753b82c0e457e975548adfb5af3a0_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195c7753b82c0e457e975548adfb5af3a0_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,710 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196470daa6b5554474b8198a04d447c49c_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196470daa6b5554474b8198a04d447c49c_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,712 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198444d64982f64fdc9393355daf67a6a5_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198444d64982f64fdc9393355daf67a6a5_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,715 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b35479b9a86449d7a9dd02d6e6ff7179_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b35479b9a86449d7a9dd02d6e6ff7179_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,717 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc2bcb25613c49a9afaca65a84e68362_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc2bcb25613c49a9afaca65a84e68362_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,720 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119da5325c4123f4122a89b87532f38746f_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119da5325c4123f4122a89b87532f38746f_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,722 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5aa6ceefba4449c9979d1844c49deb7_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e5aa6ceefba4449c9979d1844c49deb7_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,724 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e6146d044f1d4b0495389752d1a55ad6_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e6146d044f1d4b0495389752d1a55ad6_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,727 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f76829d1d94f40e69076efe08615da9b_9e9d5d81a38b9f7fdc417436276ba9a3 to hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f76829d1d94f40e69076efe08615da9b_9e9d5d81a38b9f7fdc417436276ba9a3 2024-11-19T12:54:30,727 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T12:54:30,731 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,735 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T12:54:30,737 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T12:54:30,738 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,738 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T12:54:30,738 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732020870738"}]},"ts":"9223372036854775807"} 2024-11-19T12:54:30,740 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T12:54:30,740 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9e9d5d81a38b9f7fdc417436276ba9a3, NAME => 'TestAcidGuarantees,,1732020843743.9e9d5d81a38b9f7fdc417436276ba9a3.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T12:54:30,740 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T12:54:30,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732020870740"}]},"ts":"9223372036854775807"} 2024-11-19T12:54:30,743 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T12:54:30,743 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T12:54:30,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-19T12:54:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36409 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T12:54:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T12:54:30,762 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T12:54:30,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 514) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6079 (was 6796) 2024-11-19T12:54:30,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T12:54:30,776 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:30,776 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:30,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:30,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:30,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:30,776 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T12:54:30,777 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-11-19T12:54:30,777 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=aba5a916dfea,36409,1732020693843 2024-11-19T12:54:30,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:30,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:30,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:30,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:30,843 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:30,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:30,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:30,844 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:30,844 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:30,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:30,845 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,35349,1732020694570' ***** 2024-11-19T12:54:30,845 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:54:30,845 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:30,846 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,35349,1732020694570 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;aba5a916dfea:35349. 2024-11-19T12:54:30,846 DEBUG [RS:0;aba5a916dfea:35349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:30,846 DEBUG [RS:0;aba5a916dfea:35349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:30,846 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T12:54:30,847 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T12:54:30,847 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T12:54:30,847 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:30,847 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:54:30,847 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:54:30,847 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:54:30,847 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:54:30,847 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:54:30,848 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T12:54:30,866 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:30,869 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/info/de0b88cca58f4e168360c1fa624c57d6 is 95, key is TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4./info:/1732020727187/DeleteFamily/seqid=0 2024-11-19T12:54:30,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742443_1619 (size=6514) 2024-11-19T12:54:31,048 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:31,248 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:31,273 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/info/de0b88cca58f4e168360c1fa624c57d6 2024-11-19T12:54:31,299 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/ns/2b8458fc8d034741b55452b62cb922d6 is 93, key is TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4./ns:/1732020727148/DeleteFamily/seqid=0 2024-11-19T12:54:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742444_1620 (size=5906) 2024-11-19T12:54:31,448 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:31,648 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:31,703 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/ns/2b8458fc8d034741b55452b62cb922d6 2024-11-19T12:54:31,726 INFO [regionserver/aba5a916dfea:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T12:54:31,726 INFO [regionserver/aba5a916dfea:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T12:54:31,727 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/rep_barrier/a53020a3d35d44e6a8da73d6bae83d4a is 102, key is TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4./rep_barrier:/1732020727148/DeleteFamily/seqid=0 2024-11-19T12:54:31,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742445_1621 (size=6025) 2024-11-19T12:54:31,849 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:32,049 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:32,131 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/rep_barrier/a53020a3d35d44e6a8da73d6bae83d4a 2024-11-19T12:54:32,158 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/table/fd171ac461e94b16bf5e41e2b89df6ae is 96, key is TestAcidGuarantees,,1732020699226.86dc49f9832c693612c21abbffe191c4./table:/1732020727148/DeleteFamily/seqid=0 2024-11-19T12:54:32,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742446_1622 (size=5936) 2024-11-19T12:54:32,249 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:32,449 DEBUG [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:32,562 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/table/fd171ac461e94b16bf5e41e2b89df6ae 2024-11-19T12:54:32,571 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/info/de0b88cca58f4e168360c1fa624c57d6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/info/de0b88cca58f4e168360c1fa624c57d6 2024-11-19T12:54:32,575 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/info/de0b88cca58f4e168360c1fa624c57d6, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T12:54:32,576 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/ns/2b8458fc8d034741b55452b62cb922d6 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/ns/2b8458fc8d034741b55452b62cb922d6 2024-11-19T12:54:32,579 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/ns/2b8458fc8d034741b55452b62cb922d6, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T12:54:32,580 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/rep_barrier/a53020a3d35d44e6a8da73d6bae83d4a as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/rep_barrier/a53020a3d35d44e6a8da73d6bae83d4a 2024-11-19T12:54:32,583 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/rep_barrier/a53020a3d35d44e6a8da73d6bae83d4a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T12:54:32,584 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/.tmp/table/fd171ac461e94b16bf5e41e2b89df6ae as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/table/fd171ac461e94b16bf5e41e2b89df6ae 2024-11-19T12:54:32,587 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/table/fd171ac461e94b16bf5e41e2b89df6ae, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T12:54:32,588 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false 2024-11-19T12:54:32,592 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T12:54:32,592 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:54:32,593 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:32,593 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020870847Running coprocessor pre-close hooks at 1732020870847Disabling compacts and flushes for region at 1732020870847Disabling writes for close at 1732020870847Obtaining lock to block concurrent updates at 1732020870848 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732020870848Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732020870848Flushing stores of hbase:meta,,1.1588230740 at 1732020870849 (+1 ms)Flushing 1588230740/info: creating writer at 1732020870849Flushing 1588230740/info: appending metadata at 1732020870869 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732020870869Flushing 1588230740/ns: creating writer at 1732020871281 (+412 ms)Flushing 1588230740/ns: appending metadata at 1732020871299 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732020871299Flushing 1588230740/rep_barrier: creating writer at 1732020871709 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732020871727 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732020871727Flushing 1588230740/table: creating writer at 1732020872138 (+411 ms)Flushing 1588230740/table: appending metadata at 1732020872157 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732020872157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ff1b7: reopening flushed file at 1732020872570 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@656858f5: reopening flushed file at 1732020872575 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e3447c4: reopening flushed file at 1732020872579 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7540249b: reopening flushed file at 1732020872583 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false at 1732020872588 (+5 ms)Writing region close event to WAL at 1732020872589 (+1 ms)Running coprocessor post-close hooks at 1732020872592 (+3 ms)Closed at 1732020872592 2024-11-19T12:54:32,593 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:32,650 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,35349,1732020694570; all regions closed. 2024-11-19T12:54:32,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741834_1010 (size=25344) 2024-11-19T12:54:32,660 DEBUG [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/oldWALs 2024-11-19T12:54:32,660 INFO [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C35349%2C1732020694570.meta:.meta(num 1732020698706) 2024-11-19T12:54:32,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741832_1008 (size=22929033) 2024-11-19T12:54:32,663 DEBUG [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/oldWALs 2024-11-19T12:54:32,663 INFO [RS:0;aba5a916dfea:35349 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C35349%2C1732020694570:(num 1732020697785) 2024-11-19T12:54:32,663 DEBUG [RS:0;aba5a916dfea:35349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:32,663 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:32,664 INFO [RS:0;aba5a916dfea:35349 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:32,664 INFO [RS:0;aba5a916dfea:35349 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:32,664 INFO [RS:0;aba5a916dfea:35349 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:32,664 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:32,665 INFO [RS:0;aba5a916dfea:35349 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35349 2024-11-19T12:54:32,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,35349,1732020694570 2024-11-19T12:54:32,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:32,710 INFO [RS:0;aba5a916dfea:35349 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:32,711 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,35349,1732020694570] 2024-11-19T12:54:32,757 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,35349,1732020694570 already deleted, retry=false 2024-11-19T12:54:32,757 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aba5a916dfea,35349,1732020694570 expired; onlineServers=0 2024-11-19T12:54:32,758 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'aba5a916dfea,36409,1732020693843' ***** 2024-11-19T12:54:32,758 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T12:54:32,758 INFO [M:0;aba5a916dfea:36409 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:32,758 INFO [M:0;aba5a916dfea:36409 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:32,758 DEBUG [M:0;aba5a916dfea:36409 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T12:54:32,758 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T12:54:32,758 DEBUG [M:0;aba5a916dfea:36409 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T12:54:32,759 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020697400 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020697400,5,FailOnTimeoutGroup] 2024-11-19T12:54:32,759 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020697401 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020697401,5,FailOnTimeoutGroup] 2024-11-19T12:54:32,759 INFO [M:0;aba5a916dfea:36409 {}] hbase.ChoreService(370): Chore service for: master/aba5a916dfea:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:32,759 INFO [M:0;aba5a916dfea:36409 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:32,759 DEBUG [M:0;aba5a916dfea:36409 {}] master.HMaster(1795): Stopping service threads 2024-11-19T12:54:32,759 INFO [M:0;aba5a916dfea:36409 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T12:54:32,760 INFO [M:0;aba5a916dfea:36409 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:32,760 ERROR [M:0;aba5a916dfea:36409 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35543 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35543,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T12:54:32,761 INFO [M:0;aba5a916dfea:36409 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:54:32,761 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T12:54:32,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:32,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:32,768 DEBUG [M:0;aba5a916dfea:36409 {}] zookeeper.ZKUtil(347): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T12:54:32,768 WARN [M:0;aba5a916dfea:36409 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T12:54:32,769 INFO [M:0;aba5a916dfea:36409 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/.lastflushedseqids 2024-11-19T12:54:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742447_1623 (size=72) 2024-11-19T12:54:32,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:32,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35349-0x101546ffad30001, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:32,838 INFO [RS:0;aba5a916dfea:35349 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:32,839 INFO [RS:0;aba5a916dfea:35349 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,35349,1732020694570; zookeeper connection closed. 2024-11-19T12:54:32,839 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f95a440 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f95a440 2024-11-19T12:54:32,840 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T12:54:32,958 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T12:54:33,178 INFO [M:0;aba5a916dfea:36409 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:54:33,179 INFO [M:0;aba5a916dfea:36409 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T12:54:33,179 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:54:33,180 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:33,180 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:33,180 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:54:33,180 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:33,180 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=784.84 KB heapSize=961.95 KB 2024-11-19T12:54:33,203 DEBUG [M:0;aba5a916dfea:36409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71b3a8cea4964d28ac0d432ee9559116 is 82, key is hbase:meta,,1/info:regioninfo/1732020698796/Put/seqid=0 2024-11-19T12:54:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742448_1624 (size=5672) 2024-11-19T12:54:33,607 INFO [M:0;aba5a916dfea:36409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71b3a8cea4964d28ac0d432ee9559116 2024-11-19T12:54:33,635 DEBUG [M:0;aba5a916dfea:36409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e17bcba1f9964527885ca27fc54d7680 is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1732020730377/Put/seqid=0 2024-11-19T12:54:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742449_1625 (size=45078) 2024-11-19T12:54:34,040 INFO [M:0;aba5a916dfea:36409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=784.29 KB at sequenceid=2192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e17bcba1f9964527885ca27fc54d7680 2024-11-19T12:54:34,050 INFO [M:0;aba5a916dfea:36409 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e17bcba1f9964527885ca27fc54d7680 2024-11-19T12:54:34,065 DEBUG [M:0;aba5a916dfea:36409 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d9bf7976744439be81c7cbb3c7f6c is 69, key is aba5a916dfea,35349,1732020694570/rs:state/1732020697539/Put/seqid=0 2024-11-19T12:54:34,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073742450_1626 (size=5156) 2024-11-19T12:54:34,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:54:34,193 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T12:54:34,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T12:54:34,470 INFO [M:0;aba5a916dfea:36409 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2192 (bloomFilter=true), to=hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d9bf7976744439be81c7cbb3c7f6c 2024-11-19T12:54:34,481 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71b3a8cea4964d28ac0d432ee9559116 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/71b3a8cea4964d28ac0d432ee9559116 2024-11-19T12:54:34,486 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/71b3a8cea4964d28ac0d432ee9559116, entries=8, sequenceid=2192, filesize=5.5 K 2024-11-19T12:54:34,486 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e17bcba1f9964527885ca27fc54d7680 as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e17bcba1f9964527885ca27fc54d7680 2024-11-19T12:54:34,489 INFO [M:0;aba5a916dfea:36409 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e17bcba1f9964527885ca27fc54d7680 2024-11-19T12:54:34,489 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e17bcba1f9964527885ca27fc54d7680, entries=168, sequenceid=2192, filesize=44.0 K 2024-11-19T12:54:34,490 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/975d9bf7976744439be81c7cbb3c7f6c as hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/975d9bf7976744439be81c7cbb3c7f6c 2024-11-19T12:54:34,493 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35543/user/jenkins/test-data/fffbd5bf-3611-696e-ad75-12b5f5718f27/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/975d9bf7976744439be81c7cbb3c7f6c, entries=1, sequenceid=2192, filesize=5.0 K 2024-11-19T12:54:34,493 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(3140): Finished flush of dataSize ~784.84 KB/803681, heapSize ~961.65 KB/984728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2192, compaction requested=false 2024-11-19T12:54:34,495 INFO [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:34,495 DEBUG [M:0;aba5a916dfea:36409 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020873179Disabling compacts and flushes for region at 1732020873179Disabling writes for close at 1732020873180 (+1 ms)Obtaining lock to block concurrent updates at 1732020873180Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732020873180Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=803681, getHeapSize=984968, getOffHeapSize=0, getCellsCount=2385 at 1732020873181 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732020873182 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732020873182Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732020873203 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732020873203Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732020873615 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732020873635 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732020873635Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732020874050 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732020874064 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732020874064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4d4686: reopening flushed file at 1732020874480 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ae141fd: reopening flushed file at 1732020874486 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6462a055: reopening flushed file at 1732020874490 (+4 ms)Finished flush of dataSize ~784.84 KB/803681, heapSize ~961.65 KB/984728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2192, compaction requested=false at 1732020874493 (+3 ms)Writing region close event to WAL at 1732020874495 (+2 ms)Closed at 1732020874495 2024-11-19T12:54:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741830_1006 (size=947352) 2024-11-19T12:54:34,497 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:34,498 INFO [M:0;aba5a916dfea:36409 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T12:54:34,498 INFO [M:0;aba5a916dfea:36409 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36409 2024-11-19T12:54:34,498 INFO [M:0;aba5a916dfea:36409 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:34,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:34,657 INFO [M:0;aba5a916dfea:36409 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:34,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36409-0x101546ffad30000, quorum=127.0.0.1:55548, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:34,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T12:54:34,670 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T12:54:34,670 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T12:54:34,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T12:54:34,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.log.dir/,STOPPED} 2024-11-19T12:54:34,674 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T12:54:34,674 WARN [BP-311241435-172.17.0.2-1732020690378 heartbeating to localhost/127.0.0.1:35543 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T12:54:34,674 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T12:54:34,674 WARN [BP-311241435-172.17.0.2-1732020690378 heartbeating to localhost/127.0.0.1:35543 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-311241435-172.17.0.2-1732020690378 (Datanode Uuid 68775114-ff34-4774-8f54-36651af3c6d4) service to localhost/127.0.0.1:35543 2024-11-19T12:54:34,676 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data1/current/BP-311241435-172.17.0.2-1732020690378 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T12:54:34,676 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/cluster_468353bd-3f55-3ca5-0aba-ac5cb292f07c/data/data2/current/BP-311241435-172.17.0.2-1732020690378 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T12:54:34,677 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T12:54:34,684 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T12:54:34,684 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T12:54:34,684 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T12:54:34,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T12:54:34,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a024e7fb-4709-7e1e-4c0d-ffb1c6b818e2/hadoop.log.dir/,STOPPED} 2024-11-19T12:54:34,700 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T12:54:34,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down